Merge pull request #4593 from fidencio/2.4.3-branch-bump

# Kata Containers 2.4.3

.github/cargo-deny-composite-action/cargo-deny-generator.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-#
-# Copyright (c) 2022 Red Hat
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-script_dir=$(dirname "$(readlink -f "$0")")
-parent_dir=$(realpath "${script_dir}/../..")
-cidir="${parent_dir}/ci"
-source "${cidir}/lib.sh"
-
-cargo_deny_file="${script_dir}/action.yaml"
-
-cat cargo-deny-skeleton.yaml.in > "${cargo_deny_file}"
-
-changed_files_status=$(run_get_pr_changed_file_details)
-changed_files_status=$(echo "$changed_files_status" | grep "Cargo\.toml$" || true)
-changed_files=$(echo "$changed_files_status" | awk '{print $NF}' || true)
-
-if [ -z "$changed_files" ]; then
-  cat >> "${cargo_deny_file}" << EOF
-    - run: echo "No Cargo.toml files to check"
-      shell: bash
-EOF
-fi
-
-for path in $changed_files
-do
-    cat >> "${cargo_deny_file}" << EOF
-
-    - name: ${path}
-      continue-on-error: true
-      shell: bash
-      run: |
-        pushd $(dirname ${path})
-        cargo deny check
-        popd
-EOF
-done

.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in

@@ -1,30 +0,0 @@
-#
-# Copyright (c) 2022 Red Hat
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-name: 'Cargo Crates Check'
-description: 'Checks every Cargo.toml file using cargo-deny'
-
-env:
-  CARGO_TERM_COLOR: always
-
-runs:
-  using: "composite"
-  steps:
-    - name: Install Rust
-      uses: actions-rs/toolchain@v1
-      with:
-        profile: minimal
-        toolchain: nightly 
-        override: true
-
-    - name: Cache
-      uses: Swatinem/rust-cache@v2
-
-    - name: Install Cargo deny
-      shell: bash
-      run: |
-        which cargo
-        cargo install --locked cargo-deny || true

.github/workflows/add-backport-label.yaml

@@ -1,100 +0,0 @@
-name: Add backport label
-
-on:
-  pull_request:
-    types:
-      - opened
-      - synchronize
-      - reopened
-      - edited
-      - labeled
-      - unlabeled
-
-jobs:
-  check-issues:
-    if: ${{ github.event.label.name != 'auto-backport' }}
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code to allow hub to communicate with the project
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
-
-      - name: Install hub extension script
-        run: |
-          pushd $(mktemp -d) &>/dev/null
-          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
-          sudo install hub-util.sh /usr/local/bin
-          popd &>/dev/null
-
-      - name: Determine whether to add label 
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          CONTAINS_AUTO_BACKPORT: ${{ contains(github.event.pull_request.labels.*.name, 'auto-backport') }}
-        id: add_label
-        run: |
-          pr=${{ github.event.pull_request.number }}
-          linked_issue_urls=$(hub-util.sh \
-            list-issues-for-pr "$pr" |\
-            grep -v "^\#"  |\
-            cut -d';' -f3 || true)
-          [ -z "$linked_issue_urls" ] && {
-            echo "::error::No linked issues for PR $pr"
-            exit 1
-          }
-          has_bug=false
-          for issue_url in $(echo "$linked_issue_urls")
-          do
-            issue=$(echo "$issue_url"| awk -F\/ '{print $NF}' || true)
-            [ -z "$issue" ] && {
-              echo "::error::Cannot determine issue number from $issue_url for PR $pr"
-              exit 1
-            }
-            labels=$(hub-util.sh list-labels-for-issue "$issue")
-            
-            label_names=$(echo $labels | jq -r '.[].name' || true)
-            if [[ "$label_names" =~ "bug" ]]; then
-              has_bug=true
-              break
-            fi
-          done
-
-          has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
-          has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
-
-          echo "::set-output name=add_backport_label::false"
-          if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
-            if [[ $has_no_backport_needed_label = false ]]; then
-              echo "::set-output name=add_backport_label::true"
-            fi
-          fi
-
-          # Do not spam comment, only if auto-backport label is going to be newly added.
-          echo "::set-output name=auto_backport_added::$CONTAINS_AUTO_BACKPORT"
-
-      - name: Add comment
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
-        uses: actions/github-script@v6
-        with:
-          script: |
-            github.rest.issues.createComment({
-              issue_number: context.issue.number,
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              body: 'This issue has been marked for auto-backporting. Add label(s) backport-to-BRANCHNAME to backport to them'
-            })
-
-      # Allow label to be removed by adding no-backport-needed label
-      - name: Remove auto-backport label
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'false' }}
-        uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
-        with:
-          remove-labels: "auto-backport"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Add auto-backport label
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' }}
-        uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
-        with:
-          add-labels: "auto-backport"
-          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/add-pr-sizing-label.yaml

@@ -1,40 +0,0 @@
-# Copyright (c) 2022 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-name: Add PR sizing label
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - reopened
-      - synchronize
-
-jobs:
-  add-pr-size-label:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v1
-
-      - name: Install PR sizing label script
-        run: |
-          # Clone into a temporary directory to avoid overwriting
-          # any existing github directory.
-          pushd $(mktemp -d) &>/dev/null
-          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
-          sudo install pr-add-size-label.sh /usr/local/bin
-          popd &>/dev/null
-
-      - name: Add PR sizing label
-        env:
-          GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
-        run: |
-          pr=${{ github.event.number }}
-          # Removing man-db, workflow kept failing, fixes: #4480
-          sudo apt -y remove --purge man-db
-          sudo apt -y install diffstat patchutils
-
-          pr-add-size-label.sh -p "$pr"

.github/workflows/auto-backport.yaml

@@ -1,29 +0,0 @@
-on:
-  pull_request_target:
-    types: ["labeled", "closed"]
-
-jobs:
-  backport:
-    name: Backport PR
-    runs-on: ubuntu-latest
-    if: |
-      github.event.pull_request.merged == true
-      && contains(github.event.pull_request.labels.*.name, 'auto-backport')
-      && (
-        (github.event.action == 'labeled' && github.event.label.name == 'auto-backport')
-        || (github.event.action == 'closed')
-      )
-    steps:
-      - name: Backport Action
-        uses: sqren/backport-github-action@v8.9.2
-        with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          auto_backport_label_prefix: backport-to-
-
-      - name: Info log
-        if: ${{ success() }}
-        run: cat /home/runner/.backport/backport.info.log
-        
-      - name: Debug log
-        if: ${{ failure() }}
-        run: cat /home/runner/.backport/backport.debug.log

.github/workflows/cargo-deny-runner.yaml

@@ -1,19 +0,0 @@
-name: Cargo Crates Check Runner
-on: [pull_request]
-jobs:
-  cargo-deny-runner:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout Code
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: actions/checkout@v3
-      - name: Generate Action
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: bash cargo-deny-generator.sh
-        working-directory: ./.github/cargo-deny-composite-action/
-        env:
-          GOPATH: ${{ runner.workspace }}/kata-containers
-      - name: Run Action
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        uses: ./.github/cargo-deny-composite-action

.github/workflows/cc-payload-after-push.yaml

@@ -1,98 +0,0 @@
-name: CI | Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    branches:
-      - CCv0
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cc-cloud-hypervisor
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-shim-v2
-          - cc-virtiofsd
-          - cc-sev-kernel
-          - cc-sev-ovmf
-          - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
-          - cc-tdx-qemu
-          - cc-tdx-td-shim
-          - cc-tdx-tdvf
-    steps:
-      - name: Login to Kata Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-        with:
-          fetch-depth: 0 # This is needed in order to keep the commit ids history
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-          PUSH_TO_REGISTRY: yes
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to Confidential Containers quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh $(pwd)/kata-static.tar.xz "quay.io/confidential-containers/runtime-payload-ci" "kata-containers-latest"

.github/workflows/cc-payload.yaml

@@ -1,88 +0,0 @@
-name: Publish Kata Containers payload for Confidential Containers
-on:
-  push:
-    tags:
-      - 'CC\-[0-9]+.[0-9]+.[0-9]+'
-
-jobs:
-  build-asset:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        asset:
-          - cc-cloud-hypervisor
-          - cc-kernel
-          - cc-qemu
-          - cc-rootfs-image
-          - cc-shim-v2
-          - cc-virtiofsd
-          - cc-sev-kernel
-          - cc-sev-ovmf
-          - cc-sev-rootfs-initrd
-          - cc-tdx-kernel
-          - cc-tdx-rootfs-image
-          - cc-tdx-qemu
-          - cc-tdx-td-shim
-          - cc-tdx-tdvf
-    steps:
-      - uses: actions/checkout@v3
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v3
-      - name: get-artifacts
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-          retention-days: 1
-          if-no-files-found: error
-
-  kata-payload:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - name: Login to quay.io
-        uses: docker/login-action@v2
-        with:
-          registry: quay.io
-          username: ${{ secrets.COCO_QUAY_DEPLOYER_USERNAME }}
-          password: ${{ secrets.COCO_QUAY_DEPLOYER_PASSWORD }}
-
-      - uses: actions/checkout@v3
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v3
-        with:
-          name: kata-static-tarball
-
-      - name: build-and-push-kata-payload
-        id: build-and-push-kata-payload
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-build-and-upload-payload.sh $(pwd)/kata-static.tar.xz

.github/workflows/commit-message-check.yaml

@@ -10,7 +10,7 @@ env:
   error_msg: |+
     See the document below for help on formatting commits for the project.
 
-    https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md#patch-format
+    https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md#patch-format
 
 jobs:
   commit-message-check:
@@ -47,7 +47,7 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^.{0,75}(\n.*)*$|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^.{0,75}(\n.*)*$'
         error: 'Subject too long (max 75)'
         post_error: ${{ env.error_msg }}
 
@@ -63,8 +63,7 @@ jobs:
         #   the entire commit message.
         #
         # - Body lines *can* be longer than the maximum if they start
-        #   with a non-alphabetic character or if there is no whitespace in
-        #   the line.
+        #   with a non-alphabetic character.
         #
         #   This allows stack traces, log files snippets, emails, long URLs,
         #   etc to be specified. Some of these naturally "work" as they start
@@ -75,8 +74,8 @@ jobs:
         #
         # - A SoB comment can be any length (as it is unreasonable to penalise
         #   people with long names/email addresses :)
-        pattern: '^.+(\n([a-zA-Z].{0,150}|[^a-zA-Z\n].*|[^\s\n]*|Signed-off-by:.*|))+$'
-        error: 'Body line too long (max 150)'
+        pattern: '^.+(\n([a-zA-Z].{0,149}|[^a-zA-Z\n].*|Signed-off-by:.*|))+$'
+        error: 'Body line too long (max 72)'
         post_error: ${{ env.error_msg }}
 
     - name: Check Fixes
@@ -95,6 +94,6 @@ jobs:
       uses: tim-actions/commit-message-checker-with-regex@v0.3.1
       with:
         commits: ${{ steps.get-pr-commits.outputs.commits }}
-        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:|^Merge pull request (?:kata-containers)?#[\d]+ from.*'
+        pattern: '^[\s\t]*[^:\s\t]+[\s\t]*:'
         error: 'Failed to find subsystem in subject'
         post_error: ${{ env.error_msg }}

.github/workflows/deploy-ccv0-demo.yaml

@@ -1,129 +0,0 @@
-on:
-  issue_comment:
-    types: [created, edited]
-
-name: deploy-ccv0-demo
-
-jobs:
-  check-comment-and-membership:
-    runs-on: ubuntu-latest
-    if: |
-      github.event.issue.pull_request
-      && github.event_name == 'issue_comment'
-      && github.event.action == 'created'
-      && startsWith(github.event.comment.body, '/deploy-ccv0-demo')
-    steps:
-      - name: Check membership
-        uses: kata-containers/is-organization-member@1.0.1
-        id: is_organization_member
-        with:
-          organization: kata-containers
-          username: ${{ github.event.comment.user.login }}
-          token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Fail if not member
-        run: |
-          result=${{ steps.is_organization_member.outputs.result }}
-          if [ $result == false ]; then
-              user=${{ github.event.comment.user.login }}
-              echo Either ${user} is not part of the kata-containers organization
-              echo or ${user} has its Organization Visibility set to Private at
-              echo https://github.com/orgs/kata-containers/people?query=${user}
-              echo 
-              echo Ensure you change your Organization Visibility to Public and
-              echo trigger the test again.
-              exit 1
-          fi
-
-  build-asset:
-    runs-on: ubuntu-latest
-    needs: check-comment-and-membership
-    strategy:
-      matrix:
-        asset:
-          - cloud-hypervisor
-          - firecracker
-          - kernel
-          - qemu
-          - rootfs-image
-          - rootfs-initrd
-          - shim-v2
-    steps:
-      - uses: actions/checkout@v2
-      - name: Install docker
-        run: |
-          curl -fsSL https://test.docker.com -o test-docker.sh
-          sh test-docker.sh
-
-      - name: Prepare confidential container rootfs
-        if: ${{ matrix.asset == 'rootfs-initrd' }}
-        run: |
-          pushd include_rootfs/etc
-          curl -LO https://raw.githubusercontent.com/confidential-containers/documentation/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json
-          mkdir kata-containers
-          envsubst < docs/how-to/data/confidential-agent-config.toml.in > kata-containers/agent.toml
-          popd
-        env:
-          AA_KBC_PARAMS: offline_fs_kbc::null
-
-      - name: Build ${{ matrix.asset }}
-        run: |
-          make "${KATA_ASSET}-tarball"
-          build_dir=$(readlink -f build)
-          # store-artifact does not work with symlink
-          sudo cp -r "${build_dir}" "kata-build"
-        env:
-          AA_KBC: offline_fs_kbc
-          INCLUDE_ROOTFS: include_rootfs
-          KATA_ASSET: ${{ matrix.asset }}
-          TAR_OUTPUT: ${{ matrix.asset }}.tar.gz
-
-      - name: store-artifact ${{ matrix.asset }}
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-build/kata-static-${{ matrix.asset }}.tar.xz
-          if-no-files-found: error
-
-  create-kata-tarball:
-    runs-on: ubuntu-latest
-    needs: build-asset
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-artifacts
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-artifacts
-          path: kata-artifacts
-      - name: merge-artifacts
-        run: |
-          ./tools/packaging/kata-deploy/local-build/kata-deploy-merge-builds.sh kata-artifacts
-      - name: store-artifacts
-        uses: actions/upload-artifact@v2
-        with:
-          name: kata-static-tarball
-          path: kata-static.tar.xz
-
-  kata-deploy:
-    needs: create-kata-tarball
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: get-kata-tarball
-        uses: actions/download-artifact@v2
-        with:
-          name: kata-static-tarball
-      - name: build-and-push-kata-deploy-ci
-        id: build-and-push-kata-deploy-ci
-        run: |
-          tag=$(echo $GITHUB_REF | cut -d/ -f3-)
-          pushd $GITHUB_WORKSPACE
-          git checkout $tag
-          pkg_sha=$(git rev-parse HEAD)
-          popd
-          mv kata-static.tar.xz $GITHUB_WORKSPACE/tools/packaging/kata-deploy/kata-static.tar.xz
-          docker build --build-arg KATA_ARTIFACTS=kata-static.tar.xz -t quay.io/confidential-containers/runtime-payload:$pkg_sha $GITHUB_WORKSPACE/tools/packaging/kata-deploy
-          docker login -u ${{ secrets.QUAY_DEPLOYER_USERNAME }} -p ${{ secrets.QUAY_DEPLOYER_PASSWORD }} quay.io
-          docker push quay.io/confidential-containers/runtime-payload:$pkg_sha
-          mkdir -p packaging/kata-deploy
-          ln -s $GITHUB_WORKSPACE/tools/packaging/kata-deploy/action packaging/kata-deploy/action
-          echo "::set-output name=PKG_SHA::${pkg_sha}"

.github/workflows/docs-url-alive-check.yaml

@@ -1,41 +0,0 @@
-on:
-  schedule:
-    - cron:  '0 23 * * 0'
-
-name: Docs URL Alive Check
-jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    # don't run this action on forks
-    if: github.repository_owner == 'kata-containers'
-    env:
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Set env
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    # docs url alive check
-    - name: Docs URL Alive Check
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make docs-url-alive-check

.github/workflows/kata-deploy-push.yaml

@@ -24,7 +24,6 @@ jobs:
           - firecracker
           - rootfs-image
           - rootfs-initrd
-          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/kata-deploy-test.yaml

@@ -1,10 +1,5 @@
 on:
   workflow_dispatch: # this is used to trigger the workflow on non-main branches
-    inputs:
-      pr:
-        description: 'PR number from the selected branch to test'
-        type: string
-        required: true
   issue_comment:
     types: [created, edited]
 
@@ -18,20 +13,19 @@ jobs:
       && github.event_name == 'issue_comment'
       && github.event.action == 'created'
       && startsWith(github.event.comment.body, '/test_kata_deploy')
-      || github.event_name == 'workflow_dispatch'
     steps:
-      - name: Check membership on comment or dispatch
+      - name: Check membership
         uses: kata-containers/is-organization-member@1.0.1
         id: is_organization_member
         with:
           organization: kata-containers
-          username: ${{ github.event.comment.user.login || github.event.sender.login }}
+          username: ${{ github.event.comment.user.login }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Fail if not member
         run: |
           result=${{ steps.is_organization_member.outputs.result }}
           if [ $result == false ]; then
-              user=${{ github.event.comment.user.login || github.event.sender.login }}
+              user=${{ github.event.comment.user.login }}
               echo Either ${user} is not part of the kata-containers organization
               echo or ${user} has its Organization Visibility set to Private at
               echo https://github.com/orgs/kata-containers/people?query=${user}
@@ -54,17 +48,12 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
-          - virtiofsd
     steps:
       - name: get-PR-ref
         id: get-PR-ref
         run: |
-            if [ ${{ github.event_name }} == 'issue_comment' ]; then
-                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            else # workflow_dispatch
-                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
-            fi
-            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
             echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
@@ -99,12 +88,8 @@ jobs:
       - name: get-PR-ref
         id: get-PR-ref
         run: |
-            if [ ${{ github.event_name }} == 'issue_comment' ]; then
-                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            else # workflow_dispatch
-                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
-            fi
-            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
             echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:
@@ -130,12 +115,8 @@ jobs:
       - name: get-PR-ref
         id: get-PR-ref
         run: |
-            if [ ${{ github.event_name }} == 'issue_comment' ]; then
-                ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
-            else # workflow_dispatch
-                ref="refs/pull/${{ github.event.inputs.pr }}/merge"
-            fi
-            echo "reference for PR: " ${ref} "event:" ${{ github.event_name }}
+            ref=$(cat $GITHUB_EVENT_PATH | jq -r '.issue.pull_request.url' | sed  's#^.*\/pulls#refs\/pull#' | sed 's#$#\/merge#')
+            echo "reference for PR: " ${ref}
             echo "##[set-output name=pr-ref;]${ref}"
       - uses: actions/checkout@v2
         with:

.github/workflows/release.yaml

@@ -1,8 +1,8 @@
-name: Publish Kata release artifacts
+name: Publish Kata 2.x release artifacts
 on:
   push:
     tags:
-      - '[0-9]+.[0-9]+.[0-9]+*'
+     - '2.*'
 
 jobs:
   build-asset:
@@ -17,7 +17,6 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - shim-v2
-          - virtiofsd
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/snap-release.yaml

@@ -1,9 +1,8 @@
-name: Release Kata in snapcraft store
+name: Release Kata 2.x in snapcraft store
 on:
   push:
     tags:
-      - '[0-9]+.[0-9]+.[0-9]+*'
-
+      - '2.*'
 jobs:
   release-snap:
     runs-on: ubuntu-20.04
@@ -20,8 +19,6 @@ jobs:
 
       - name: Build snap
         run: |
-          # Removing man-db, workflow kept failing, fixes: #4480
-          sudo apt -y remove --purge man-db
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
           latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)

.github/workflows/static-checks.yaml

@@ -8,7 +8,7 @@ on:
 
 name: Static checks
 jobs:
-  check-vendored-code:
+  test:
     strategy:
       matrix:
         go-version: [1.16.x, 1.17.x]
@@ -24,7 +24,7 @@ jobs:
     steps:
     - name: Install Go
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
+      uses: actions/setup-go@v2
       with:
         go-version: ${{ matrix.go-version }}
       env:
@@ -43,7 +43,7 @@ jobs:
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
+      uses: actions/checkout@v2
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
@@ -58,303 +58,39 @@ jobs:
         cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Installing rust
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
+        PATH=$PATH:"$HOME/.cargo/bin"
+        rustup target add x86_64-unknown-linux-musl
+        rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     # Check whether the vendored code is up-to-date & working as the first thing
     - name: Check vendored code
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
-
-  static-checks:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Static Checks
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
-
-  compiler-checks:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: ${{ matrix.go-version }}
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Run Compiler Checks
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
-
-  unit-tests:
-    runs-on: ubuntu-20.04
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.17.x
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Run Unit Tests
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
-
-  unit-tests-as-root:
-    runs-on: ubuntu-20.04
-    env:
-      TRAVIS: "true"
-      TRAVIS_BRANCH: ${{ github.base_ref }}
-      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
-      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
-      RUST_BACKTRACE: "1"
-      target_branch: ${{ github.base_ref }}
-    steps:
-    - name: Install Go
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v3
-      with:
-        go-version: 1.17.x 
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Setup GOPATH
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
-        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
-        echo "TRAVIS: ${TRAVIS}"
-    - name: Set env
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
-    - name: Checkout code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-        path: ./src/github.com/${{ github.repository }}
-    - name: Setup travis references
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
-        target_branch=${TRAVIS_BRANCH}
-    - name: Setup
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
-      env:
-        GOPATH: ${{ runner.workspace }}/kata-containers
-    - name: Installing rust
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
-        PATH=$PATH:"$HOME/.cargo/bin"
-        rustup target add x86_64-unknown-linux-musl
-        rustup component add rustfmt clippy
-    - name: Setup seccomp
-      run: |
-        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
-        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
-        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
-        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
-        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Run Unit Tests As Root User
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && sudo -E PATH="$PATH" make test
-
-  test-dragonball:
-    runs-on: self-hosted
-    env:
-      RUST_BACKTRACE: "1"
-    steps:
-      - uses: actions/checkout@v3
-      - name: Set env
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
-      - name: Install Rust
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          ./ci/install_rust.sh
-          PATH=$PATH:"$HOME/.cargo/bin"
-      - name: Run Unit Test
-        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-        run: |
-          cd src/dragonball
-          /root/.cargo/bin/cargo version
-          rustc --version
-          sudo -E env PATH=$PATH LIBC=gnu SUPPORT_VIRTUALIZATION=true make test

.gitignore

@@ -10,5 +10,4 @@ src/agent/kata-agent.service
 src/agent/protocols/src/*.rs
 !src/agent/protocols/src/lib.rs
 build
-src/tools/log-parser/kata-log-parser
 

Makefile

@@ -6,25 +6,24 @@
 # List of available components
 COMPONENTS =
 
-COMPONENTS += libs
 COMPONENTS += agent
-COMPONENTS += dragonball
 COMPONENTS += runtime
-COMPONENTS += runtime-rs
 
 # List of available tools
 TOOLS =
 
 TOOLS += agent-ctl
-TOOLS += kata-ctl
-TOOLS += log-parser
-TOOLS += runk
 TOOLS += trace-forwarder
 
 STANDARD_TARGETS = build check clean install test vendor
 
 default: all
 
+all: logging-crate-tests build
+
+logging-crate-tests:
+	make -C src/libs/logging
+
 include utils.mk
 include ./tools/packaging/kata-deploy/local-build/Makefile
 
@@ -40,15 +39,10 @@ generate-protocols:
 static-checks: build
 	bash ci/static-checks.sh
 
-docs-url-alive-check:
-	bash ci/docs-url-alive-check.sh
-
 .PHONY: \
 	all \
 	binary-tarball \
 	default \
 	install-binary-tarball \
-	static-checks \
-	docs-url-alive-check
-
-
+	logging-crate-tests \
+	static-checks

README.md

@@ -71,7 +71,6 @@ See the [official documentation](docs) including:
 - [Developer guide](docs/Developer-Guide.md)
 - [Design documents](docs/design)
   - [Architecture overview](docs/design/architecture)
-  - [Architecture 3.0 overview](docs/design/architecture_3.0/)
 
 ## Configuration
 
@@ -117,9 +116,7 @@ The table below lists the core parts of the project:
 | Component | Type | Description |
 |-|-|-|
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
-| [runtime-rs](src/runtime-rs) | core | The Rust version runtime. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
-| [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads |
 | [documentation](docs) | documentation | Documentation common to all components (such as design and install documentation). |
 | [tests](https://github.com/kata-containers/tests) | tests | Excludes unit tests which live with the main code. |
 
@@ -133,9 +130,7 @@ The table below lists the remaining parts of the project:
 | [kernel](https://www.kernel.org) | kernel | Linux kernel used by the hypervisor to boot the guest image. Patches are stored [here](tools/packaging/kernel). |
 | [osbuilder](tools/osbuilder) | infrastructure | Tool to create "mini O/S" rootfs and initrd images and kernel for the hypervisor. |
 | [`agent-ctl`](src/tools/agent-ctl) | utility | Tool that provides low-level access for testing the agent. |
-| [`kata-ctl`](src/tools/kata-ctl) | utility | Tool that provides advanced commands and debug facilities. |
 | [`trace-forwarder`](src/tools/trace-forwarder) | utility | Agent tracing helper. |
-| [`runk`](src/tools/runk) | utility | Standard OCI container runtime based on the agent. |
 | [`ci`](https://github.com/kata-containers/ci) | CI | Continuous Integration configuration files and scripts. |
 | [`katacontainers.io`](https://github.com/kata-containers/www.katacontainers.io) | Source for the [`katacontainers.io`](https://www.katacontainers.io) site. |
 
@@ -143,7 +138,7 @@ The table below lists the remaining parts of the project:
 
 Kata Containers is now
 [available natively for most distributions](docs/install/README.md#packaged-installation-methods).
-However, packaging scripts and metadata are still used to generate [snap](snap/local) and GitHub releases. See
+However, packaging scripts and metadata are still used to generate snap and GitHub releases. See
 the [components](#components) section for further details.
 
 ## Glossary of Terms

VERSION

@@ -1 +1 @@
-3.1.0-alpha0
+2.4.3

ci/darwin-test.sh

@@ -11,10 +11,10 @@ runtimedir=$cidir/../src/runtime
 
 build_working_packages() {
 	# working packages:
-	device_api=$runtimedir/pkg/device/api
-	device_config=$runtimedir/pkg/device/config
-	device_drivers=$runtimedir/pkg/device/drivers
-	device_manager=$runtimedir/pkg/device/manager
+	device_api=$runtimedir/virtcontainers/device/api
+	device_config=$runtimedir/virtcontainers/device/config
+	device_drivers=$runtimedir/virtcontainers/device/drivers
+	device_manager=$runtimedir/virtcontainers/device/manager
 	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
 	utils_pkg_dir=$runtimedir/virtcontainers/utils
 

ci/go-test.sh

@@ -1,6 +1,6 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
-# Copyright (c) 2021 Easystack Inc.
+# Copyright (c) 2020 Intel Corporation
 #
 # SPDX-License-Identifier: Apache-2.0
 
@@ -9,4 +9,4 @@ set -e
 cidir=$(dirname "$0")
 source "${cidir}/lib.sh"
 
-run_docs_url_alive_check
+run_go_test

ci/install_libseccomp.sh

@@ -19,31 +19,29 @@ source "${tests_repo_dir}/.ci/lib.sh"
 # fail. So let's ensure they are unset here.
 unset PREFIX DESTDIR
 
-arch=${ARCH:-$(uname -m)}
+arch=$(uname -m)
 workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 
 # Variables for libseccomp
-libseccomp_version="${LIBSECCOMP_VERSION:-""}"
-if [ -z "${libseccomp_version}" ]; then
-    libseccomp_version=$(get_version "externals.libseccomp.version")
-fi
-libseccomp_url="${LIBSECCOMP_URL:-""}"
-if [ -z "${libseccomp_url}" ]; then
-    libseccomp_url=$(get_version "externals.libseccomp.url")
-fi
+# Currently, specify the libseccomp version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# libseccomp_version=$(get_version "externals.libseccomp.version")
+# libseccomp_url=$(get_version "externals.libseccomp.url")
+libseccomp_version="2.5.1"
+libseccomp_url="https://github.com/seccomp/libseccomp"
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
 cflags="-O2"
 
 # Variables for gperf
-gperf_version="${GPERF_VERSION:-""}"
-if [ -z "${gperf_version}" ]; then
-    gperf_version=$(get_version "externals.gperf.version")
-fi
-gperf_url="${GPERF_URL:-""}"
-if [ -z "${gperf_url}" ]; then
-    gperf_url=$(get_version "externals.gperf.url")
-fi
+# Currently, specify the gperf version directly without using `versions.yaml`
+# because the current Snap workflow is incomplete.
+# After solving the issue, replace this code by using the `versions.yaml`.
+# gperf_version=$(get_version "externals.gperf.version")
+# gperf_url=$(get_version "externals.gperf.url")
+gperf_version="3.1"
+gperf_url="https://ftp.gnu.org/gnu/gperf"
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"
 
@@ -72,9 +70,7 @@ build_and_install_gperf() {
     curl -sLO "${gperf_tarball_url}"
     tar -xf "${gperf_tarball}"
     pushd "gperf-${gperf_version}"
-    # gperf is a build time dependency of libseccomp and not to be used in the target.
-    # Unset $CC since that might point to a cross compiler.
-    CC= ./configure --prefix="${gperf_install_dir}"
+    ./configure --prefix="${gperf_install_dir}"
     make
     make install
     export PATH=$PATH:"${gperf_install_dir}"/bin
@@ -88,7 +84,7 @@ build_and_install_libseccomp() {
     curl -sLO "${libseccomp_tarball_url}"
     tar -xf "${libseccomp_tarball}"
     pushd "libseccomp-${libseccomp_version}"
-    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static --host="${arch}"
+    ./configure --prefix="${libseccomp_install_dir}" CFLAGS="${cflags}" --enable-static
     make
     make install
     popd

ci/install_musl.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# Copyright (c) 2020 Ant Group
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -e
+
+install_aarch64_musl() {
+	local arch=$(uname -m)
+	if [ "${arch}" == "aarch64" ]; then
+		local musl_tar="${arch}-linux-musl-native.tgz"
+		local musl_dir="${arch}-linux-musl-native"
+		pushd /tmp
+		if curl -sLO --fail https://musl.cc/${musl_tar}; then
+			tar -zxf ${musl_tar}
+			mkdir -p /usr/local/musl/
+			cp -r ${musl_dir}/* /usr/local/musl/
+		fi
+		popd
+	fi
+}
+
+install_aarch64_musl

ci/lib.sh

@@ -18,13 +18,6 @@ clone_tests_repo()
 {
 	if [ -d "$tests_repo_dir" ]; then
 		[ -n "${CI:-}" ] && return
-		# git config --global --add safe.directory will always append
-		# the target to .gitconfig without checking the existence of
-		# the target, so it's better to check it before adding the target repo.
-		local sd="$(git config --global --get safe.directory ${tests_repo_dir} || true)"
-		if [ -z "${sd}" ]; then
-			git config --global --add safe.directory ${tests_repo_dir}
-		fi
 		pushd "${tests_repo_dir}"
 		git checkout "${branch}"
 		git pull
@@ -46,21 +39,8 @@ run_static_checks()
 	bash "$tests_repo_dir/.ci/static-checks.sh" "$@"
 }
 
-run_docs_url_alive_check()
+run_go_test()
 {
 	clone_tests_repo
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers"
-}
-
-run_get_pr_changed_file_details()
-{
-	clone_tests_repo
-	# Make sure we have the targeting branch
-	git remote set-branches --add origin "${branch}"
-	git fetch -a
-	source "$tests_repo_dir/.ci/lib.sh"
-	get_pr_changed_file_details
+	bash "$tests_repo_dir/.ci/go-test.sh"
 }

deny.toml

@@ -1,33 +0,0 @@
-targets = [
-    { triple = "x86_64-apple-darwin" },
-    { triple = "x86_64-unknown-linux-gnu" },
-    { triple = "x86_64-unknown-linux-musl" },
-]
-
-[advisories]
-vulnerability = "deny"
-unsound = "deny"
-unmaintained = "deny"
-ignore = ["RUSTSEC-2020-0071"]
-
-[bans]
-multiple-versions = "allow"
-deny = [
-    { name = "cmake" },
-    { name = "openssl-sys" },
-]
-
-[licenses]
-unlicensed = "deny"
-allow-osi-fsf-free = "neither"
-copyleft = "allow"
-# We want really high confidence when inferring licenses from text
-confidence-threshold = 0.93
-allow = ["0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "CC0-1.0", "ISC", "MIT", "MPL-2.0"]
-private = { ignore = true}
-
-exceptions = []
-
-[sources]
-unknown-registry = "allow"
-unknown-git = "allow"

docs/Developer-Guide.md

@@ -33,41 +33,51 @@ You need to install the following to build Kata Containers components:
 - `make`.
 - `gcc` (required for building the shim and runtime).
 
-# Build and install Kata Containers
-## Build and install the Kata Containers runtime
+# Build and install the Kata Containers runtime
 
-```bash
-$ git clone https://github.com/kata-containers/kata-containers.git
-$ pushd kata-containers/src/runtime
-$ make && sudo -E "PATH=$PATH" make install
-$ sudo mkdir -p /etc/kata-containers/
-$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
-$ popd
+```
+$ go get -d -u github.com/kata-containers/kata-containers
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/runtime
+$ make && sudo -E PATH=$PATH make install
 ```
 
 The build will create the following:
 
 - runtime binary: `/usr/local/bin/kata-runtime` and `/usr/local/bin/containerd-shim-kata-v2`
-- configuration file: `/usr/share/defaults/kata-containers/configuration.toml` and `/etc/kata-containers/configuration.toml`
+- configuration file: `/usr/share/defaults/kata-containers/configuration.toml`
+
+# Check hardware requirements
+
+You can check if your system is capable of creating a Kata Container by running the following:
+
+```
+$ sudo kata-runtime check
+```
+
+If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
 
 ## Configure to use initrd or rootfs image
 
 Kata containers can run with either an initrd image or a rootfs image.
 
-If you want to test with `initrd`, make sure you have uncommented `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
-in your configuration file, commenting out the `image` line in
-`/etc/kata-containers/configuration.toml`. For example:
+If you want to test with `initrd`, make sure you have `initrd = /usr/share/kata-containers/kata-containers-initrd.img`
+in your configuration file, commenting out the `image` line:
 
-```bash
+`/usr/share/defaults/kata-containers/configuration.toml` and comment out the `image` line with the following. For example:
+
+```
+$ sudo mkdir -p /etc/kata-containers/
+$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
 $ sudo sed -i 's/^\(image =.*\)/# \1/g' /etc/kata-containers/configuration.toml
-$ sudo sed -i 's/^# \(initrd =.*\)/\1/g' /etc/kata-containers/configuration.toml
 ```
 You can create the initrd image as shown in the [create an initrd image](#create-an-initrd-image---optional) section.
 
-If you want to test with a rootfs `image`, make sure you have uncommented `image = /usr/share/kata-containers/kata-containers.img`
+If you want to test with a rootfs `image`, make sure you have `image = /usr/share/kata-containers/kata-containers.img`
 in your configuration file, commenting out the `initrd` line. For example:
 
-```bash
+```
+$ sudo mkdir -p /etc/kata-containers/
+$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
 $ sudo sed -i 's/^\(initrd =.*\)/# \1/g' /etc/kata-containers/configuration.toml
 ```
 The rootfs image is created as shown in the [create a rootfs image](#create-a-rootfs-image) section.
@@ -80,7 +90,7 @@ rootfs `image`(100MB+).
 
 Enable seccomp as follows:
 
-```bash
+```
 $ sudo sed -i '/^disable_guest_seccomp/ s/true/false/' /etc/kata-containers/configuration.toml
 ```
 
@@ -90,7 +100,9 @@ This will pass container seccomp profiles to the kata agent.
 
 Enable full debug as follows:
 
-```bash
+```
+$ sudo mkdir -p /etc/kata-containers/
+$ sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
 $ sudo sed -i -e 's/^# *\(enable_debug\).*=.*$/\1 = true/g' /etc/kata-containers/configuration.toml
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.log=debug initcall_debug"/g' /etc/kata-containers/configuration.toml
 ```
@@ -104,7 +116,7 @@ detailed below.
 The Kata logs appear in the `containerd` log files, along with logs from `containerd` itself.
 
 For more information about `containerd` debug, please see the
-[`containerd` documentation](https://github.com/containerd/containerd/blob/main/docs/getting-started.md).
+[`containerd` documentation](https://github.com/containerd/containerd/blob/master/docs/getting-started.md).
 
 #### Enabling full `containerd` debug
 
@@ -163,7 +175,7 @@ and offers possible workarounds and fixes.
 it stores. When messages are suppressed, it is noted in the logs. This can be checked
 for by looking for those notifications, such as:
 
-```bash
+```sh
 $ sudo journalctl --since today | fgrep Suppressed
 Jun 29 14:51:17 mymachine systemd-journald[346]: Suppressed 4150 messages from /system.slice/docker.service
 ```
@@ -188,7 +200,7 @@ RateLimitBurst=0
 
 Restart `systemd-journald` for the changes to take effect:
 
-```bash
+```sh
 $ sudo systemctl restart systemd-journald
 ```
 
@@ -202,24 +214,25 @@ $ sudo systemctl restart systemd-journald
 
 The agent is built with a statically linked `musl.` The default `libc` used is `musl`, but on `ppc64le` and `s390x`, `gnu` should be used. To configure this:
 
-```bash
-$ export ARCH="$(uname -m)"
+```
+$ export ARCH=$(uname -m)
 $ if [ "$ARCH" = "ppc64le" -o "$ARCH" = "s390x" ]; then export LIBC=gnu; else export LIBC=musl; fi
-$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
-$ rustup target add "${ARCH}-unknown-linux-${LIBC}"
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ rustup target add ${ARCH}-unknown-linux-${LIBC}
 ```
 
 To build the agent:
 
-```bash
-$ make -C kata-containers/src/agent
+```
+$ go get -d -u github.com/kata-containers/kata-containers
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/src/agent && make
 ```
 
 The agent is built with seccomp capability by default.
 If you want to build the agent without the seccomp capability, you need to run `make` with `SECCOMP=no` as follows.
 
-```bash
-$ make -C kata-containers/src/agent SECCOMP=no
+```
+$ make -C $GOPATH/src/github.com/kata-containers/kata-containers/src/agent SECCOMP=no
 ```
 
 > **Note:**
@@ -227,6 +240,13 @@ $ make -C kata-containers/src/agent SECCOMP=no
 > - If you enable seccomp in the main configuration file but build the agent without seccomp capability,
 >   the runtime exits conservatively with an error message.
 
+## Get the osbuilder
+
+```
+$ go get -d -u github.com/kata-containers/kata-containers
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder
+```
+
 ## Create a rootfs image
 ### Create a local rootfs
 
@@ -234,26 +254,24 @@ As a prerequisite, you need to install Docker. Otherwise, you will not be
 able to run the `rootfs.sh` script with `USE_DOCKER=true` as expected in
 the following example.
 
-```bash
-$ export distro="ubuntu" # example
-$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
-$ sudo rm -rf "${ROOTFS_DIR}"
-$ pushd kata-containers/tools/osbuilder/rootfs-builder
-$ script -fec 'sudo -E USE_DOCKER=true ./rootfs.sh "${distro}"'
-$ popd
+```
+$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
+$ sudo rm -rf ${ROOTFS_DIR}
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true ./rootfs.sh ${distro}'
 ```
 
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```bash
-$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
+```
+$ ./rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
-```bash
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
 
 > **Note:**
@@ -269,18 +287,17 @@ $ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${
 >
 > - You should only do this step if you are testing with the latest version of the agent.
 
-```bash
-$ sudo install -o root -g root -m 0550 -t "${ROOTFS_DIR}/usr/bin" "${ROOTFS_DIR}/../../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent"
-$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-agent.service" "${ROOTFS_DIR}/usr/lib/systemd/system/"
-$ sudo install -o root -g root -m 0440 "${ROOTFS_DIR}/../../../../src/agent/kata-containers.target" "${ROOTFS_DIR}/usr/lib/systemd/system/"
+```
+$ sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ../../../src/agent/target/x86_64-unknown-linux-musl/release/kata-agent
+$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
+$ sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
 ```
 
 ### Build a rootfs image
 
-```bash
-$ pushd  kata-containers/tools/osbuilder/image-builder
-$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh "${ROOTFS_DIR}"'
-$ popd
+```
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/image-builder
+$ script -fec 'sudo -E USE_DOCKER=true ./image_builder.sh ${ROOTFS_DIR}'
 ```
 
 > **Notes:**
@@ -296,26 +313,21 @@ $ popd
 
 ### Install the rootfs image
 
-```bash
-$ pushd kata-containers/tools/osbuilder/image-builder
-$ commit="$(git log --format=%h -1 HEAD)"
-$ date="$(date +%Y-%m-%d-%T.%N%z)"
+```
+$ commit=$(git log --format=%h -1 HEAD)
+$ date=$(date +%Y-%m-%d-%T.%N%z)
 $ image="kata-containers-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
-$ popd
 ```
 
 ## Create an initrd image - OPTIONAL
 ### Create a local rootfs for initrd image
-
-```bash
-$ export distro="ubuntu" # example
-$ export ROOTFS_DIR="$(realpath kata-containers/tools/osbuilder/rootfs-builder/rootfs)"
-$ sudo rm -rf "${ROOTFS_DIR}"
-$ pushd kata-containers/tools/osbuilder/rootfs-builder/
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh "${distro}"'
-$ popd
+```
+$ export ROOTFS_DIR="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs"
+$ sudo rm -rf ${ROOTFS_DIR}
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true ./rootfs.sh ${distro}'
 ```
 `AGENT_INIT` controls if the guest image uses the Kata agent as the guest `init` process. When you create an initrd image,
 always set `AGENT_INIT` to `yes`.
@@ -323,14 +335,14 @@ always set `AGENT_INIT` to `yes`.
 You MUST choose a distribution (e.g., `ubuntu`) for `${distro}`.
 You can get a supported distributions list in the Kata Containers by running the following.
 
-```bash
-$ ./kata-containers/tools/osbuilder/rootfs-builder/rootfs.sh -l
+```
+$ ./rootfs.sh -l
 ```
 
 If you want to build the agent without seccomp capability, you need to run the `rootfs.sh` script with `SECCOMP=no` as follows.
 
-```bash
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${distro}"'
+```
+$ script -fec 'sudo -E GOPATH=$GOPATH AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh ${distro}'
 ```
 
 > **Note:**
@@ -339,31 +351,28 @@ $ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true SECCOMP=no ./rootfs.sh "${
 
 Optionally, add your custom agent binary to the rootfs with the following commands. The default `$LIBC` used
 is `musl`, but on ppc64le and s390x, `gnu` should be used. Also, Rust refers to ppc64le as `powerpc64le`:
-```bash
-$ export ARCH="$(uname -m)"
-$ [ "${ARCH}" == "ppc64le" ] || [ "${ARCH}" == "s390x" ] && export LIBC=gnu || export LIBC=musl
-$ [ "${ARCH}" == "ppc64le" ] && export ARCH=powerpc64le
-$ sudo install -o root -g root -m 0550 -T "${ROOTFS_DIR}/../../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent" "${ROOTFS_DIR}/sbin/init"
+```
+$ export ARCH=$(uname -m)
+$ [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
+$ [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
+$ sudo install -o root -g root -m 0550 -T ../../../src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent ${ROOTFS_DIR}/sbin/init
 ```
 
 ### Build an initrd image
 
-```bash
-$ pushd kata-containers/tools/osbuilder/initrd-builder
-$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh "${ROOTFS_DIR}"'
-$ popd
+```
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/initrd-builder
+$ script -fec 'sudo -E AGENT_INIT=yes USE_DOCKER=true ./initrd_builder.sh ${ROOTFS_DIR}'
 ```
 
 ### Install the initrd image
 
-```bash
-$ pushd kata-containers/tools/osbuilder/initrd-builder
-$ commit="$(git log --format=%h -1 HEAD)"
-$ date="$(date +%Y-%m-%d-%T.%N%z)"
+```
+$ commit=$(git log --format=%h -1 HEAD)
+$ date=$(date +%Y-%m-%d-%T.%N%z)
 $ image="kata-containers-initrd-${date}-${commit}"
 $ sudo install -o root -g root -m 0640 -D kata-containers-initrd.img "/usr/share/kata-containers/${image}"
 $ (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers-initrd.img)
-$ popd
 ```
 
 # Install guest kernel images
@@ -382,43 +391,43 @@ Kata Containers makes use of upstream QEMU branch. The exact version
 and repository utilized can be found by looking at the [versions file](../versions.yaml).
 
 Find the correct version of QEMU from the versions file:
-```bash
-$ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_version="$(get_from_kata_deps "assets.hypervisor.qemu.version")"
-$ echo "${qemu_version}"
+```
+$ source ${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/scripts/lib.sh
+$ qemu_version=$(get_from_kata_deps "assets.hypervisor.qemu.version")
+$ echo ${qemu_version}
 ```
 Get source from the matching branch of QEMU:
-```bash
-$ git clone -b "${qemu_version}" https://github.com/qemu/qemu.git
-$ your_qemu_directory="$(realpath qemu)"
+```
+$ go get -d github.com/qemu/qemu
+$ cd ${GOPATH}/src/github.com/qemu/qemu
+$ git checkout ${qemu_version}
+$ your_qemu_directory=${GOPATH}/src/github.com/qemu/qemu
 ```
 
 There are scripts to manage the build and packaging of QEMU. For the examples below, set your
 environment as:
-```bash
-$ packaging_dir="$(realpath kata-containers/tools/packaging)"
+```
+$ go get -d github.com/kata-containers/kata-containers
+$ packaging_dir="${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging"
 ```
 
 Kata often utilizes patches for not-yet-upstream and/or backported fixes for components,
 including QEMU. These can be found in the [packaging/QEMU directory](../tools/packaging/qemu/patches),
 and it's *recommended* that you apply them. For example, suppose that you are going to build QEMU
 version 5.2.0, do:
-```bash
-$ "$packaging_dir/scripts/apply_patches.sh" "$packaging_dir/qemu/patches/5.2.x/"
+```
+$ cd $your_qemu_directory
+$ $packaging_dir/scripts/apply_patches.sh $packaging_dir/qemu/patches/5.2.x/
 ```
 
 To build utilizing the same options as Kata, you should make use of the `configure-hypervisor.sh` script. For example:
-```bash
-$ pushd "$your_qemu_directory"
-$ "$packaging_dir/scripts/configure-hypervisor.sh" kata-qemu > kata.cfg
-$ eval ./configure "$(cat kata.cfg)"
-$ make -j $(nproc --ignore=1)
-# Optional
-$ sudo -E make install
-$ popd
 ```
-
-If you do not want to install the respective QEMU version, the configuration file can be modified to point to the correct binary. In `/etc/kata-containers/configuration.toml`, change `path = "/path/to/qemu/build/qemu-system-x86_64"` to point to the correct QEMU binary.
+$ cd $your_qemu_directory
+$ $packaging_dir/scripts/configure-hypervisor.sh kata-qemu > kata.cfg
+$ eval ./configure "$(cat kata.cfg)"
+$ make -j $(nproc)
+$ sudo -E make install
+```
 
 See the [static-build script for QEMU](../tools/packaging/static-build/qemu/build-static-qemu.sh) for a reference on how to get, setup, configure and build QEMU for Kata.
 
@@ -430,33 +439,11 @@ See the [static-build script for QEMU](../tools/packaging/static-build/qemu/buil
 >   under upstream review for supporting NVDIMM on aarch64.
 >
 You could build the custom `qemu-system-aarch64` as required with the following command:
-```bash
-$ git clone https://github.com/kata-containers/tests.git
-$ script -fec 'sudo -E tests/.ci/install_qemu.sh'
 ```
-
-## Build `virtiofsd`
-
-When using the file system type virtio-fs (default), `virtiofsd` is required
-
-```bash
-$ pushd kata-containers/tools/packaging/static-build/virtiofsd
-$ ./build-static-virtiofsd.sh
-$ popd
+$ go get -d github.com/kata-containers/tests
+$ script -fec 'sudo -E ${GOPATH}/src/github.com/kata-containers/tests/.ci/install_qemu.sh'
 ```
 
-Modify `/etc/kata-containers/configuration.toml` and update value `virtio_fs_daemon = "/path/to/kata-containers/tools/packaging/static-build/virtiofsd/virtiofsd/virtiofsd"` to point to the binary.
-
-# Check hardware requirements
-
-You can check if your system is capable of creating a Kata Container by running the following:
-
-```bash
-$ sudo kata-runtime check
-```
-
-If your system is *not* able to run Kata Containers, the previous command will error out and explain why.
-
 # Run Kata Containers with Containerd
 Refer to the [How to use Kata Containers and Containerd](how-to/containerd-kata.md) how-to guide.
 
@@ -478,7 +465,7 @@ script and paste its output directly into a
 > [runtime](../src/runtime) repository.
 
 To perform analysis on Kata logs, use the
-[`kata-log-parser`](../src/tools/log-parser)
+[`kata-log-parser`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
 tool, which can convert the logs into formats (e.g. JSON, TOML, XML, and YAML).
 
 See [Set up a debug console](#set-up-a-debug-console).
@@ -487,7 +474,7 @@ See [Set up a debug console](#set-up-a-debug-console).
 
 ## Checking Docker default runtime
 
-```bash
+```
 $ sudo docker info 2>/dev/null | grep -i "default runtime" | cut -d: -f2- | grep -q runc  && echo "SUCCESS" || echo "ERROR: Incorrect default Docker runtime"
 ```
 ## Set up a debug console
@@ -504,7 +491,7 @@ contain either `/bin/sh` or `/bin/bash`.
 
 Enable debug_console_enabled in the `configuration.toml` configuration file:
 
-```toml
+```
 [agent.kata]
 debug_console_enabled = true
 ```
@@ -515,7 +502,7 @@ This will pass `agent.debug_console agent.debug_console_vport=1026` to agent as
 
 For Kata Containers `2.0.x` releases, the `kata-runtime exec` command depends on the`kata-monitor` running, in order to get the sandbox's `vsock` address to connect to. Thus, first start the `kata-monitor` process.
 
-```bash
+```
 $ sudo kata-monitor
 ```
 
@@ -535,7 +522,7 @@ bash-4.2# exit
 exit
 ```
 
-`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/main/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
+`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
  with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/).
 For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option.
 
@@ -577,10 +564,10 @@ an additional `coreutils` package.
 
 For example using CentOS:
 
-```bash
-$ pushd kata-containers/tools/osbuilder/rootfs-builder
-$ export ROOTFS_DIR="$(realpath ./rootfs)"
-$ script -fec 'sudo -E USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
+```
+$ cd $GOPATH/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder
+$ export ROOTFS_DIR=${GOPATH}/src/github.com/kata-containers/kata-containers/tools/osbuilder/rootfs-builder/rootfs
+$ script -fec 'sudo -E GOPATH=$GOPATH USE_DOCKER=true EXTRA_PKGS="bash coreutils" ./rootfs.sh centos'
 ```
 
 #### Build the debug image
@@ -595,10 +582,9 @@ Install the image:
 >**Note**: When using an initrd image, replace the below rootfs image name `kata-containers.img` 
 >with the initrd image name `kata-containers-initrd.img`.
 
-```bash
+```
 $ name="kata-containers-centos-with-debug-console.img"
 $ sudo install -o root -g root -m 0640 kata-containers.img "/usr/share/kata-containers/${name}"
-$ popd
 ```
 
 Next, modify the `image=` values in the `[hypervisor.qemu]` section of the
@@ -607,7 +593,7 @@ to specify the full path to the image name specified in the previous code
 section. Alternatively, recreate the symbolic link so it points to
 the new debug image:
 
-```bash
+```
 $ (cd /usr/share/kata-containers && sudo ln -sf "$name" kata-containers.img)
 ```
 
@@ -618,7 +604,7 @@ to avoid all subsequently created containers from using the debug image.
 
 Create a container as normal. For example using `crictl`:
 
-```bash
+```
 $ sudo crictl run -r kata container.yaml pod.yaml
 ```
 
@@ -631,7 +617,7 @@ those for firecracker / cloud-hypervisor.
 
 Add `agent.debug_console` to the guest kernel command line to allow the agent process to start a debug console. 
 
-```bash
+```
 $ sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console"/g' "${kata_configuration_file}"
 ```
 
@@ -652,7 +638,7 @@ between the host and the guest. The kernel command line option `agent.debug_cons
 
 Add the parameter `agent.debug_console_vport=1026` to the kernel command line
 as shown below:
-```bash
+```
 sudo sed -i -e 's/^kernel_params = "\(.*\)"/kernel_params = "\1 agent.debug_console_vport=1026"/g' "${kata_configuration_file}"
 ```
 
@@ -665,7 +651,7 @@ Next, connect to the debug console. The VSOCKS paths vary slightly between each
 VMM solution.
 
 In case of cloud-hypervisor, connect to the `vsock` as shown:
-```bash
+```
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id}/root/ && socat stdin unix-connect:clh.sock'
 CONNECT 1026
 ```
@@ -673,7 +659,7 @@ CONNECT 1026
 **Note**: You need to type `CONNECT 1026` and press `RETURN` key after entering the `socat` command.
 
 For firecracker, connect to the `hvsock` as shown:
-```bash
+```
 $ sudo su -c 'cd /var/run/vc/firecracker/${sandbox_id}/root/ && socat stdin unix-connect:kata.hvsock'
 CONNECT 1026
 ```
@@ -682,7 +668,7 @@ CONNECT 1026
 
 
 For QEMU, connect to the `vsock` as shown:
-```bash
+```
 $ sudo su -c 'cd /var/run/vc/vm/${sandbox_id} && socat "stdin,raw,echo=0,escape=0x11" "unix-connect:console.sock"'
 ```
 
@@ -695,7 +681,7 @@ If the image is created using
 [osbuilder](../tools/osbuilder), the following YAML
 file exists and contains details of the image and how it was created:
 
-```bash
+```
 $ cat /var/lib/osbuilder/osbuilder.yaml
 ```
 
@@ -714,11 +700,11 @@ options to have the kernel boot messages logged into the system journal.
 For generic information on enabling debug in the configuration file, see the
 [Enable full debug](#enable-full-debug) section.
 
-The kernel boot messages will appear in the `kata` logs (and in the `containerd` or `CRI-O` log appropriately).
+The kernel boot messages will appear in the `containerd` or `CRI-O` log appropriately,
 such as:
 
 ```bash
-$ sudo journalctl -t kata
+$ sudo journalctl -t containerd
 -- Logs begin at Thu 2020-02-13 16:20:40 UTC, end at Thu 2020-02-13 16:30:23 UTC. --
 ...
 time="2020-09-15T14:56:23.095113803+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.395399] brd: module loaded"
@@ -728,4 +714,3 @@ time="2020-09-15T14:56:23.105268162+08:00" level=debug msg="reading guest consol
 time="2020-09-15T14:56:23.121121598+08:00" level=debug msg="reading guest console" console-protocol=unix console-url=/run/vc/vm/ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791/console.sock pid=107642 sandbox=ab9f633385d4987828d342e47554fc6442445b32039023eeddaa971c1bb56791 source=virtcontainers subsystem=sandbox vmconsole="[    0.421324] memmap_init_zone_device initialised 32768 pages in 12ms"
 ...
 ```
-Refer to the [kata-log-parser documentation](../src/tools/log-parser/README.md) which is useful to fetch these.

docs/Limitations.md

@@ -46,7 +46,7 @@ The following link shows the latest list of limitations:
 # Contributing
 
 If you would like to work on resolving a limitation, please refer to the
-[contributors guide](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).
+[contributors guide](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md).
 If you wish to raise an issue for a new limitation, either
 [raise an issue directly on the runtime](https://github.com/kata-containers/kata-containers/issues/new)
 or see the
@@ -60,26 +60,17 @@ This section lists items that might be possible to fix.
 ## OCI CLI commands
 
 ### Docker and Podman support
-Currently Kata Containers does not support Podman.
+Currently Kata Containers does not support Docker or Podman.
 
 See issue https://github.com/kata-containers/kata-containers/issues/722 for more information.
 
-Docker supports Kata Containers since 22.06:
-
-```bash
-$ sudo docker run --runtime io.containerd.kata.v2
-```
-
-Kata Containers works perfectly with containerd, we recommend to use
-containerd's Docker-style command line tool [`nerdctl`](https://github.com/containerd/nerdctl).
-
 ## Runtime commands
 
 ### checkpoint and restore
 
 The runtime does not provide `checkpoint` and `restore` commands. There
 are discussions about using VM save and restore to give us a
-[`criu`](https://github.com/checkpoint-restore/criu)-like functionality,
+`[criu](https://github.com/checkpoint-restore/criu)`-like functionality,
 which might provide a solution.
 
 Note that the OCI standard does not specify `checkpoint` and `restore`
@@ -102,42 +93,6 @@ All other configurations are supported and are working properly.
 
 ## Networking
 
-### Host network
-
-Host network (`nerdctl/docker run --net=host`or [Kubernetes `HostNetwork`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#hosts-namespaces)) is not supported.
-It is not possible to directly access the host networking configuration
-from within the VM.
-
-The `--net=host` option can still be used with `runc` containers and
-inter-mixed with running Kata Containers, thus enabling use of `--net=host`
-when necessary.
-
-It should be noted, currently passing the `--net=host` option into a
-Kata Container may result in the Kata Container networking setup
-modifying, re-configuring and therefore possibly breaking the host
-networking setup. Do not use `--net=host` with Kata Containers.
-
-### Support for joining an existing VM network
-
-Docker supports the ability for containers to join another containers
-namespace with the `docker run --net=containers` syntax. This allows
-multiple containers to share a common network namespace and the network
-interfaces placed in the network namespace. Kata Containers does not
-support network namespace sharing. If a Kata Container is setup to
-share the network namespace of a `runc` container, the runtime
-effectively takes over all the network interfaces assigned to the
-namespace and binds them to the VM. Consequently, the `runc` container loses
-its network connectivity.
-
-### docker run --link
-
-The runtime does not support the `docker run --link` command. This
-command is now deprecated by docker and we have no intention of adding support.
-Equivalent functionality can be achieved with the newer docker networking commands.
-
-See more documentation at
-[docs.docker.com](https://docs.docker.com/network/links/).
-
 ## Resource management
 
 Due to the way VMs differ in their CPU and memory allocation, and sharing

docs/README.md

@@ -30,6 +30,7 @@ See the [how-to documentation](how-to).
 * [GPU Passthrough with Kata](./use-cases/GPU-passthrough-and-Kata.md)
 * [SR-IOV with Kata](./use-cases/using-SRIOV-and-kata.md)
 * [Intel QAT with Kata](./use-cases/using-Intel-QAT-and-kata.md)
+* [VPP with Kata](./use-cases/using-vpp-and-kata.md)
 * [SPDK vhost-user with Kata](./use-cases/using-SPDK-vhostuser-and-kata.md)
 * [Intel SGX with Kata](./use-cases/using-Intel-SGX-and-kata.md)
 

docs/Unit-Test-Advice.md

@@ -277,9 +277,7 @@ mod tests {
 
 ## Temporary files
 
-Use `t.TempDir()` to create temporary directory. The directory created by
-`t.TempDir()` is automatically removed when the test and all its subtests
-complete.
+Always delete temporary files on success.
 
 ### Golang temporary files
 
@@ -288,7 +286,11 @@ func TestSomething(t *testing.T) {
     assert := assert.New(t)
 
     // Create a temporary directory
-    tmpdir := t.TempDir()
+    tmpdir, err := os.MkdirTemp("", "")
+    assert.NoError(err)
+
+    // Delete it at the end of the test
+    defer os.RemoveAll(tmpdir) 
 
     // Add test logic that will use the tmpdir here...
 }
@@ -320,7 +322,7 @@ mod tests {
 
 ## Test user
 
-[Unit tests are run *twice*](../src/runtime/go-test.sh):
+[Unit tests are run *twice*](https://github.com/kata-containers/tests/blob/main/.ci/go-test.sh):
 
 - as the current user
 - as the `root` user (if different to the current user)
@@ -341,7 +343,7 @@ The main repository has the most comprehensive set of skip abilities. See:
 
 One method is to use the `nix` crate along with some custom macros:
 
-```rust
+```
 #[cfg(test)]
 mod tests {
     #[allow(unused_macros)]

docs/code-pr-advice.md

@@ -79,7 +79,7 @@ a "`BUG: feature X not implemented see {bug-url}`" type error.
 - Don't use multiple log calls when a single log call could be used.
 
 - Use structured logging where possible to allow
-  [standard tooling](../src/tools/log-parser)
+  [standard tooling](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
   be able to extract the log fields.
 
 ### Names

docs/design/README.md

@@ -11,8 +11,7 @@ Kata Containers design documents:
 - [`Inotify` support](inotify.md)
 - [Metrics(Kata 2.0)](kata-2-0-metrics.md)
 - [Design for Kata Containers `Lazyload` ability with `nydus`](kata-nydus-design.md)
-- [Design for direct-assigned volume](direct-blk-device-assignment.md)
-- [Design for core-scheduling](core-scheduling.md)
+
 ---
 
 - [Design proposals](proposals)

docs/design/VSocks.md

@@ -67,15 +67,22 @@ Using a proxy for multiplexing the connections between the VM and the host uses
 4.5MB per [POD][2]. In a high density deployment this could add up to GBs of
 memory that could have been used to host more PODs. When we talk about density
 each kilobyte matters and it might be the decisive factor between run another
-POD or not. Before making the decision not to use VSOCKs, you should ask
+POD or not. For example if you have 500 PODs running in a server, the same
+amount of [`kata-proxy`][3] processes will be running and consuming for around
+2250MB of RAM. Before making the decision not to use VSOCKs, you should ask
 yourself, how many more containers can run with the memory RAM consumed by the
 Kata proxies?
 
 ### Reliability
 
+[`kata-proxy`][3] is in charge of multiplexing the connections between virtual
+machine and host processes, if it dies all connections get broken. For example
+if you have a [POD][2] with 10 containers running, if `kata-proxy` dies it would
+be impossible to contact your containers, though they would still be running.
 Since communication via VSOCKs is direct, the only way to lose communication
 with the containers is if the VM itself or the `containerd-shim-kata-v2` dies, if this happens
 the containers are removed automatically.
 
 [1]: https://wiki.qemu.org/Features/VirtioVsock
 [2]: ./vcpu-handling.md#virtual-cpus-and-kubernetes-pods
+[3]: https://github.com/kata-containers/proxy

docs/design/architecture/kubernetes.md

@@ -17,7 +17,7 @@ Kubelet instance is responsible for managing the lifecycle of pods
 within the nodes and eventually relies on a container runtime to
 handle execution. The Kubelet architecture decouples lifecycle
 management from container execution through a dedicated gRPC based
-[Container Runtime Interface (CRI)](https://github.com/kubernetes/design-proposals-archive/blob/main/node/container-runtime-interface-v1.md).
+[Container Runtime Interface (CRI)](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/node/container-runtime-interface-v1.md).
 
 In other words, a Kubelet is a CRI client and expects a CRI
 implementation to handle the server side of the interface.

docs/design/architecture/storage.md

@@ -1,17 +1,5 @@
 # Storage
 
-## Limits
-
-Kata Containers is [compatible](README.md#compatibility) with existing
-standards and runtime. From the perspective of storage, this means no
-limits are placed on the amount of storage a container
-[workload](README.md#workload) may use.
-
-Since cgroups are not able to set limits on storage allocation, if you
-wish to constrain the amount of storage a container uses, consider
-using an existing facility such as `quota(1)` limits or
-[device mapper](#devicemapper) limits.
-
 ## virtio SCSI
 
 If a block-based graph driver is [configured](README.md#configuration),
@@ -32,7 +20,7 @@ For virtio-fs, the [runtime](README.md#runtime) starts one `virtiofsd` daemon
 ## Devicemapper
 
 The
-[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/main/snapshots/devmapper)
+[devicemapper `snapshotter`](https://github.com/containerd/containerd/tree/master/snapshots/devmapper)
 is a special case. The `snapshotter` uses dedicated block devices
 rather than formatted filesystems, and operates at the block level
 rather than the file level. This knowledge is used to directly use the

docs/design/architecture_3.0/README.md

@@ -1,169 +0,0 @@
-# Kata 3.0 Architecture
-## Overview
-In cloud-native scenarios, there is an increased demand for container startup speed, resource consumption, stability, and security, areas where the present Kata Containers runtime is challenged relative to other runtimes. To achieve this, we propose a solid, field-tested and secure Rust version of the kata-runtime.
-
-Also, we provide the following designs:
-
-- Turn key solution with builtin `Dragonball` Sandbox
-- Async I/O to reduce resource consumption
-- Extensible framework for multiple services, runtimes and hypervisors
-- Lifecycle management for sandbox and container associated resources
-
-### Rationale for choosing Rust
-
-We chose Rust because it is designed as a system language with a focus on efficiency.
-In contrast to Go, Rust makes a variety of design trade-offs in order to obtain
-good execution performance, with innovative techniques that, in contrast to C or
-C++, provide reasonable protection against common memory errors (buffer
-overflow, invalid pointers, range errors), error checking (ensuring errors are
-dealt with), thread safety, ownership of resources, and more.
-
-These benefits were verified in our project when the Kata Containers guest agent
-was rewritten in Rust. We notably saw a significant reduction in memory usage
-with the Rust-based implementation.
-
-
-## Design
-### Architecture
-![architecture](./images/architecture.png)
-### Built-in VMM
-#### Current Kata 2.x architecture
-![not_builtin_vmm](./images/not_built_in_vmm.png)
-As shown in the figure, runtime and VMM are separate processes. The runtime process forks the VMM process and interacts through the inter-process RPC. Typically, process interaction consumes more resources than peers within the process, and it will result in relatively low efficiency. At the same time, the cost of resource operation and maintenance should be considered. For example, when performing resource recovery under abnormal conditions, the exception of any process must be detected by others and activate the appropriate resource recovery process. If there are additional processes, the recovery becomes even more difficult.
-#### How To Support Built-in VMM
-We provide `Dragonball` Sandbox to enable built-in VMM by integrating VMM's function into the Rust library. We could perform VMM-related functionalities by using the library. Because runtime and VMM  are in the same process, there is a benefit in terms of message processing speed and API synchronization. It can also guarantee the consistency of the runtime and the VMM life cycle, reducing resource recovery and exception handling maintenance, as shown in the figure:
-![builtin_vmm](./images/built_in_vmm.png)
-### Async Support
-#### Why Need Async
-**Async is already in stable Rust and allows us to write async code**
-
-- Async provides significantly reduced CPU and memory overhead, especially for workloads with a large amount of IO-bound tasks
-- Async is zero-cost in Rust, which means that you only pay for what you use. Specifically, you can use async without heap allocations and dynamic dispatch, which greatly improves efficiency
-- For more (see [Why Async?](https://rust-lang.github.io/async-book/01_getting_started/02_why_async.html) and [The State of Asynchronous Rust](https://rust-lang.github.io/async-book/01_getting_started/03_state_of_async_rust.html)).
-
-**There may be several problems if implementing kata-runtime with Sync Rust**
-
-- Too many threads with a new TTRPC connection
-   - TTRPC threads: reaper thread(1) + listener thread(1) + client handler(2)
-- Add 3 I/O threads with a new container
-- In Sync mode, implementing a timeout mechanism is challenging. For example, in TTRPC API interaction, the timeout mechanism is difficult to align with Golang
-#### How To Support Async
-The kata-runtime is controlled by TOKIO_RUNTIME_WORKER_THREADS to run the OS thread, which is 2 threads by default. For TTRPC and container-related threads run in the `tokio` thread in a unified manner, and related dependencies need to be switched to Async, such as Timer, File, Netlink, etc. With the help of Async, we can easily support no-block I/O and timer. Currently, we only utilize Async for kata-runtime. The built-in VMM keeps the OS thread because it can ensure that the threads are controllable.
-
-**For N tokio worker threads and M containers**
-
-- Sync runtime(both OS thread and `tokio` task are OS thread but without `tokio` worker thread)  OS thread number:  4 + 12*M
-- Async runtime(only OS thread is OS thread) OS thread number: 2 + N
-```shell
-├─ main(OS thread)
-├─ async-logger(OS thread)
-└─ tokio worker(N * OS thread)
-  ├─ agent log forwarder(1 * tokio task)
-  ├─ health check thread(1 * tokio task)
-  ├─ TTRPC reaper thread(M * tokio task)
-  ├─ TTRPC listener thread(M * tokio task)
-  ├─ TTRPC client handler thread(7 * M * tokio task)
-  ├─ container stdin io thread(M * tokio task)
-  ├─ container stdout io thread(M * tokio task)
-  └─ container stderr io thread(M * tokio task)
-```
-### Extensible Framework
-The Kata 3.x runtime is designed with the extension of service, runtime, and hypervisor, combined with configuration to meet the needs of different scenarios. At present, the service provides a register mechanism to support multiple services. Services could interact with runtime through messages. In addition, the runtime handler handles messages from services. To meet the needs of a binary that supports multiple runtimes and hypervisors, the startup must obtain the runtime handler type and hypervisor type through configuration.
-
-![framework](./images/framework.png)
-### Resource Manager
-In our case, there will be a variety of resources, and every resource has several subtypes. Especially for `Virt-Container`, every subtype of resource has different operations. And there may be dependencies, such as the share-fs rootfs and the share-fs volume will use share-fs resources to share files to the VM. Currently, network and share-fs are regarded as sandbox resources, while rootfs, volume, and cgroup are regarded as container resources. Also, we abstract a common interface for each resource and use subclass operations to evaluate the differences between different subtypes.
-![resource manager](./images/resourceManager.png)
-
-## Roadmap
-
-- Stage 1 (June): provide basic features (current delivered)
-- Stage 2 (September): support common features
-- Stage 3: support full features
-
-| **Class**                  | **Sub-Class**       | **Development Stage** | **Status** |
-| -------------------------- | ------------------- | --------------------- |------------|
-| Service                    | task service        | Stage 1               |  ✅        |
-|                            | extend service      | Stage 3               |  🚫        |
-|                            | image service       | Stage 3               |  🚫        |
-| Runtime handler            | `Virt-Container`    | Stage 1               |  ✅        |
-| Endpoint                   | VETH Endpoint       | Stage 1               |  ✅        |
-|                            | Physical Endpoint   | Stage 2               |  ✅        |
-|                            | Tap Endpoint        | Stage 2               |  ✅        |
-|                            | `Tuntap` Endpoint   | Stage 2               |  ✅        |
-|                            | `IPVlan` Endpoint   | Stage 2               |  ✅        |
-|                            | `MacVlan` Endpoint  | Stage 2               |  ✅        |
-|                            | MACVTAP Endpoint    | Stage 3               |  🚫        |
-|                            | `VhostUserEndpoint` | Stage 3               |  🚫        |
-| Network Interworking Model | Tc filter           | Stage 1               |  ✅        |
-|                            | `MacVtap`           | Stage 3               |  🚧        |
-| Storage                    | Virtio-fs           | Stage 1               |  ✅        |
-|                            | `nydus`             | Stage 2               |  🚧        |
-|                            | `device mapper`     | Stage 2               |  🚫        |
-| `Cgroup V2`                |                     | Stage 2               |  🚧        |
-| Hypervisor                 | `Dragonball`        | Stage 1               |  🚧        |
-|                            | QEMU                | Stage 2               |  🚫        |
-|                            | ACRN                | Stage 3               |  🚫        |
-|                            | Cloud Hypervisor    | Stage 3               |  🚫        |
-|                            | Firecracker         | Stage 3               |  🚫        |
-
-## FAQ
-
-- Are the "service", "message dispatcher" and "runtime handler" all part of the single Kata 3.x runtime binary?
-
-  Yes. They are components in Kata 3.x runtime. And they will be packed into one binary.
-  1. Service is an interface, which is responsible for handling multiple services like task service, image service and etc. 
-  2. Message dispatcher, it is used to match multiple requests from the service module. 
-  3. Runtime handler is used to deal with the operation for sandbox and container. 
-- What is the name of the Kata 3.x runtime binary?
-  
-  Apparently we can't use `containerd-shim-v2-kata` because it's already used. We are facing the hardest issue of "naming" again. Any suggestions are welcomed.
-  Internally we use `containerd-shim-v2-rund`.
-
-- Is the Kata 3.x design compatible with the containerd shimv2 architecture? 
-  
-  Yes. It is designed to follow the functionality of go version kata.  And it implements the `containerd shim v2` interface/protocol.
-
-- How will users migrate to the Kata 3.x architecture?
-  
-  The migration plan will be provided before the Kata 3.x is merging into the main branch.
-
-- Is `Dragonball` limited to its own built-in VMM? Can the `Dragonball` system be configured to work using an external `Dragonball` VMM/hypervisor?
-
-  The `Dragonball` could work as an external hypervisor. However, stability and performance is challenging in this case. Built in VMM could optimise the container overhead, and it's easy to maintain stability.
-
-  `runD` is the `containerd-shim-v2` counterpart of `runC` and can run a pod/containers. `Dragonball` is a `microvm`/VMM that is designed to run container workloads. Instead of `microvm`/VMM, we sometimes refer to it as secure sandbox.
-
-- QEMU, Cloud Hypervisor and Firecracker support are planned, but how that would work. Are they working in separate process?
- 
-  Yes. They are unable to work as built in VMM.
-
-- What is `upcall`?
-  
-    The `upcall` is used to hotplug CPU/memory/MMIO devices, and it solves two issues.
-    1. avoid dependency on PCI/ACPI
-    2. avoid dependency on `udevd` within guest and get deterministic results for hotplug operations. So `upcall` is an alternative to ACPI based CPU/memory/device hotplug. And we may cooperate with the community to add support for ACPI based CPU/memory/device hotplug if needed.
-   
-    `Dbs-upcall` is a `vsock-based` direct communication tool between VMM and guests. The server side of the `upcall` is a driver in guest kernel (kernel patches are needed for this feature) and it'll start to serve the requests once the kernel has started. And the client side is in VMM , it'll be a thread that communicates with VSOCK through `uds`. We have accomplished device hotplug / hot-unplug directly through `upcall` in order to avoid virtualization of ACPI  to minimize virtual machine's overhead. And there could be many other usage through this direct communication channel. It's already open source.
-   https://github.com/openanolis/dragonball-sandbox/tree/main/crates/dbs-upcall 
-
-- The URL below says the kernel patches work with 4.19, but do they also work with 5.15+ ?
-  
-  Forward compatibility should be achievable, we have ported it to 5.10 based kernel.
-
-- Are these patches platform-specific or would they work for any architecture that supports VSOCK?
-  
-  It's almost platform independent, but some message related to CPU hotplug are platform dependent.
-
-- Could the kernel driver be replaced with a userland daemon in the guest using loopback VSOCK?
-  
-  We need to create device nodes for hot-added CPU/memory/devices, so it's not easy for userspace daemon to do these tasks.
-
-- The fact that `upcall` allows communication between the VMM and the guest suggests that this architecture might be incompatible with https://github.com/confidential-containers where the VMM should have no knowledge of what happens inside the VM.
-  
-  1. `TDX` doesn't support CPU/memory hotplug yet.
-  2. For ACPI based device hotplug, it depends on ACPI `DSDT` table, and the guest kernel will execute `ASL` code to handle during handling those hotplug event. And it should be easier to audit VSOCK based communication than ACPI `ASL` methods.
-
-- What is the security boundary for the monolithic / "Built-in VMM" case?
-  
-  It has the security boundary of virtualization. More details will be provided in next stage.

docs/design/core-scheduling.md

@@ -1,12 +0,0 @@
-# Core scheduling
-
-Core scheduling is a Linux kernel feature that allows only trusted tasks to run concurrently on
-CPUs sharing compute resources (for example, hyper-threads on a core).
-
-Containerd versions >= 1.6.4 leverage this to treat all of the processes associated with a
-given pod or container to be a single group of trusted tasks. To indicate this should be carried
-out, containerd sets the `SCHED_CORE` environment variable for each shim it spawns. When this is
-set, the Kata Containers shim implementation uses the `prctl` syscall to create a new core scheduling
-domain for the shim process itself as well as future VMM processes it will start.
-
-For more details on the core scheduling feature, see the [Linux documentation](https://www.kernel.org/doc/html/latest/admin-guide/hw-vuln/core-scheduling.html).

docs/design/direct-blk-device-assignment.md

@@ -1,253 +0,0 @@
-# Motivation
-Today, there exist a few gaps between Container Storage Interface (CSI) and virtual machine (VM) based runtimes such as Kata Containers 
-that prevent them from working together smoothly.
-
-First, it’s cumbersome to use a persistent volume (PV) with Kata Containers. Today, for a PV with Filesystem volume mode, Virtio-fs
-is the only way to surface it inside a Kata Container guest VM. But often mounting the filesystem (FS) within the guest operating system (OS) is 
-desired due to performance benefits, availability of native FS features and security benefits over the Virtio-fs mechanism.
-
-Second, it’s difficult if not impossible to resize a PV online with Kata Containers. While a PV can be expanded on the host OS, 
-the updated metadata needs to be propagated to the guest OS in order for the application container to use the expanded volume. 
-Currently, there is not a way to propagate the PV metadata from the host OS to the guest OS without restarting the Pod sandbox.
-
-# Proposed Solution
-
-Because of the OS boundary, these features cannot be implemented in the CSI node driver plugin running on the host OS 
-as is normally done in the runc container. Instead, they can be done by the Kata Containers agent inside the guest OS, 
-but it requires the CSI driver to pass the relevant information to the Kata Containers runtime. 
-An ideal long term solution would be to have the `kubelet` coordinating the communication between the CSI driver and 
-the container runtime, as described in [KEP-2857](https://github.com/kubernetes/enhancements/pull/2893/files). 
-However, as the KEP is still under review, we would like to propose a short/medium term solution to unblock our use case.
-
-The proposed solution is built on top of a previous [proposal](https://github.com/egernst/kata-containers/blob/da-proposal/docs/design/direct-assign-volume.md) 
-described by Eric Ernst. The previous proposal has two gaps:
-
-1. Writing a `csiPlugin.json` file to the volume root path introduced a security risk. A malicious user can gain unauthorized 
-access to a block device by writing their own `csiPlugin.json` to the above location through an ephemeral CSI plugin.  
-
-2. The proposal didn't describe how to establish a mapping between a volume and a kata sandbox, which is needed for 
-implementing CSI volume resize and volume stat collection APIs.
-
-This document particularly focuses on how to address these two gaps.
-
-## Assumptions and Limitations
-1. The proposal assumes that a block device volume will only be used by one Pod on a node at a time, which we believe 
-is the most common pattern in Kata Containers use cases. It’s also unsafe to have the same block device attached to more than 
-one Kata pod. In the context of Kubernetes, the `PersistentVolumeClaim` (PVC) needs to have the `accessMode` as `ReadWriteOncePod`. 
-2. More advanced Kubernetes volume features such as, `fsGroup`, `fsGroupChangePolicy`, and `subPath` are not supported. 
-
-## End User Interface
-
-1. The user specifies a PV as a direct-assigned volume. How a PV is specified as a direct-assigned volume is left for each CSI implementation to decide.
-There are a few options for reference:
-   1. A storage class parameter specifies whether it's a direct-assigned volume. This avoids any lookups of PVC 
-   or Pod information from the CSI plugin (as external provisioner takes care of these). However, all PVs in the storage class with the parameter set 
-   will have host mounts skipped.
-   2. Use a PVC annotation. This approach requires the CSI plugins have `--extra-create-metadata` [set](https://kubernetes-csi.github.io/docs/external-provisioner.html#persistentvolumeclaim-and-persistentvolume-parameters)
-   to be able to perform a lookup of the PVC annotations from the API server. Pro: API server lookup of annotations only required during creation of PV. 
-   Con: The CSI plugin will always skip host mounting of the PV.
-   3. The CSI plugin can also lookup pod `runtimeclass` during `NodePublish`. This approach can be found in the [ALIBABA CSI plugin](https://github.com/kubernetes-sigs/alibaba-cloud-csi-driver/blob/master/pkg/disk/nodeserver.go#L248).
-2. The CSI node driver delegates the direct assigned volume to the Kata Containers runtime. The CSI node driver APIs need to 
-   be modified to pass the volume mount information and collect volume information to/from the Kata Containers runtime by invoking `kata-runtime` command line commands.
-   * **NodePublishVolume** -- It invokes `kata-runtime direct-volume add --volume-path [volumePath] --mount-info [mountInfo]` 
-   to propagate the volume mount information to the Kata Containers runtime for it to carry out the filesystem mount operation.
-   The `volumePath` is the [target_path](https://github.com/container-storage-interface/spec/blob/master/csi.proto#L1364) in the CSI `NodePublishVolumeRequest`.
-   The `mountInfo` is a serialized JSON string. 
-   * **NodeGetVolumeStats** -- It invokes `kata-runtime direct-volume stats --volume-path [volumePath]` to retrieve the filesystem stats of direct-assigned volume.
-   * **NodeExpandVolume** -- It invokes `kata-runtime direct-volume resize --volume-path [volumePath] --size [size]` to send a resize request to the Kata Containers runtime to
-   resize the direct-assigned volume.
-   * **NodeStageVolume/NodeUnStageVolume** -- It invokes `kata-runtime direct-volume remove --volume-path [volumePath]` to remove the persisted metadata of a direct-assigned volume.
-
-The `mountInfo` object is defined as follows:
-```Golang
-type MountInfo struct {
-    // The type of the volume (ie. block)
-    VolumeType string `json:"volume-type"`
-    // The device backing the volume.
-    Device string `json:"device"`
-    // The filesystem type to be mounted on the volume.
-    FsType string `json:"fstype"`
-    // Additional metadata to pass to the agent regarding this volume.
-    Metadata map[string]string `json:"metadata,omitempty"`
-    // Additional mount options.
-    Options []string `json:"options,omitempty"`
-}
-```
-Notes: given that the `mountInfo` is persisted to the disk by the Kata runtime, it shouldn't container any secrets (such as SMB mount password).
-
-## Implementation Details
-
-### Kata runtime
-Instead of the CSI node driver writing the mount info into a `csiPlugin.json` file under the volume root, 
-as described in the original proposal, here we propose that the CSI node driver passes the mount information to 
-the Kata Containers runtime through a new `kata-runtime` commandline command. The `kata-runtime` then writes the mount 
-information to a `mount-info.json` file in a predefined location (`/run/kata-containers/shared/direct-volumes/[volume_path]/`).
-
-When the Kata Containers runtime starts a container, it verifies whether a volume mount is a direct-assigned volume by checking 
-whether there is a `mountInfo` file under the computed Kata `direct-volumes` directory. If it is, the runtime parses the `mountInfo` file, 
-updates the mount spec with the data in `mountInfo`. The updated mount spec is then passed to the Kata agent in the guest VM together
-with other mounts. The Kata Containers runtime also creates a file named by the sandbox id under the `direct-volumes/[volume_path]/` 
-directory. The reason for adding a sandbox id file is to establish a mapping between the volume and the sandbox using it. 
-Later, when the Kata Containers runtime handles the `get-stats` and `resize` commands, it uses the sandbox id to identify 
-the endpoint of the corresponding `containerd-shim-kata-v2`.
-
-### containerd-shim-kata-v2 changes
-`containerd-shim-kata-v2` provides an API for sandbox management through a Unix domain socket. Two new handlers are proposed: `/direct-volume/stats` and `/direct-volume/resize`:
-
-Example:
-
-```bash
-$ curl --unix-socket "$shim_socket_path" -I -X GET 'http://localhost/direct-volume/stats/[urlSafeVolumePath]'
-$ curl --unix-socket "$shim_socket_path" -I -X POST 'http://localhost/direct-volume/resize' -d '{ "volumePath"": [volumePath], "Size": "123123" }'
-```
-
-The shim then forwards the corresponding request to the `kata-agent` to carry out the operations inside the guest VM. For `resize` operation, 
-the Kata runtime also needs to notify the hypervisor to resize the block device (e.g. call `block_resize` in QEMU). 
-
-### Kata agent changes
-
-The mount spec of a direct-assigned volume is passed to `kata-agent` through the existing `Storage` GRPC object. 
-Two new APIs and three new GRPC objects are added to GRPC protocol between the shim and agent for resizing and getting volume stats:
-```protobuf
-
-rpc GetVolumeStats(VolumeStatsRequest) returns (VolumeStatsResponse);
-rpc ResizeVolume(ResizeVolumeRequest) returns (google.protobuf.Empty);
-
-message VolumeStatsRequest {
-// The volume path on the guest outside the container
-    string volume_guest_path = 1;
-}
-
-message ResizeVolumeRequest {
-// Full VM guest path of the volume (outside the container)
-    string volume_guest_path = 1;
-    uint64 size = 2;
-}
-
-// This should be kept in sync with CSI NodeGetVolumeStatsResponse (https://github.com/container-storage-interface/spec/blob/v1.5.0/csi.proto)
-message VolumeStatsResponse {
-   // This field is OPTIONAL.
-   repeated VolumeUsage usage = 1;
-   // Information about the current condition of the volume.
-   // This field is OPTIONAL.
-   // This field MUST be specified if the VOLUME_CONDITION node
-   // capability is supported.
-   VolumeCondition volume_condition = 2;
-}
-message VolumeUsage {
-   enum Unit {
-      UNKNOWN = 0;
-      BYTES = 1;
-      INODES = 2;
-   }
-   // The available capacity in specified Unit. This field is OPTIONAL.
-   // The value of this field MUST NOT be negative.
-   uint64 available = 1;
-
-   // The total capacity in specified Unit. This field is REQUIRED.
-   // The value of this field MUST NOT be negative.
-   uint64 total = 2;
-
-   // The used capacity in specified Unit. This field is OPTIONAL.
-   // The value of this field MUST NOT be negative.
-   uint64 used = 3;
-
-   // Units by which values are measured. This field is REQUIRED.
-   Unit unit = 4;
-}
-
-// VolumeCondition represents the current condition of a volume.
-message VolumeCondition {
-
-   // Normal volumes are available for use and operating optimally.
-   // An abnormal volume does not meet these criteria.
-   // This field is REQUIRED.
-   bool abnormal = 1;
-
-   // The message describing the condition of the volume.
-   // This field is REQUIRED.
-   string message = 2;
-}
-
-```
-
-### Step by step walk-through
-
-Given the following definition:
-```YAML
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: app
-spec:
-  runtime-class: kata-qemu
-  containers:
-  - name: app
-    image: centos
-    command: ["/bin/sh"]
-    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
-    volumeMounts:
-    - name: persistent-storage
-      mountPath: /data
-  volumes:
-  - name: persistent-storage
-    persistentVolumeClaim:
-      claimName: ebs-claim
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  annotations:
-    skip-hostmount: "true"
-  name: ebs-claim
-spec:
-  accessModes:
-    - ReadWriteOncePod
-  volumeMode: Filesystem
-  storageClassName: ebs-sc
-  resources:
-    requests:
-      storage: 4Gi
----
-kind: StorageClass
-apiVersion: storage.k8s.io/v1
-metadata:
-  name: ebs-sc
-provisioner: ebs.csi.aws.com
-volumeBindingMode: WaitForFirstConsumer
-parameters:
-  csi.storage.k8s.io/fstype: ext4
-
-```
-Let’s assume that changes have been made in the `aws-ebs-csi-driver` node driver.
-
-**Node publish volume**
-1. In the node CSI driver, the `NodePublishVolume` API invokes: `kata-runtime direct-volume add --volume-path "/kubelet/a/b/c/d/sdf" --mount-info "{\"Device\": \"/dev/sdf\", \"fstype\": \"ext4\"}"`.
-2. The `Kata-runtime` writes the mount-info JSON to a file called `mountInfo.json` under `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
-
-**Node unstage volume**
-1. In the node CSI driver, the `NodeUnstageVolume` API invokes: `kata-runtime direct-volume remove --volume-path "/kubelet/a/b/c/d/sdf"`.
-2. Kata-runtime deletes the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
-
-**Use the volume in sandbox**
-1. Upon the request to start a container, the `containerd-shim-kata-v2` examines the container spec,
-and iterates through the mounts. For each mount, if there is a `mountInfo.json` file under `/run/kata-containers/shared/direct-volumes/[mount source path]`,
-it generates a `storage` GRPC object after overwriting the mount spec with the information in `mountInfo.json`.
-2. The shim sends the storage objects to kata-agent through TTRPC.
-3. The shim writes a file with the sandbox id as the name under `/run/kata-containers/shared/direct-volumes/[mount source path]`.
-4. The kata-agent mounts the storage objects for the container.
-
-**Node expand volume**
-1. In the node CSI driver, the `NodeExpandVolume` API invokes: `kata-runtime direct-volume resize –-volume-path "/kubelet/a/b/c/d/sdf" –-size 8Gi`.
-2. The Kata runtime checks whether there is a sandbox id file under the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
-3. The Kata runtime identifies the shim instance through the sandbox id, and sends a GRPC request to resize the volume.
-4. The shim handles the request, asks the hypervisor to resize the block device and sends a GRPC request to Kata agent to resize the filesystem.
-5. Kata agent receives the request and resizes the filesystem.
-
-**Node get volume stats**
-1. In the node CSI driver, the `NodeGetVolumeStats` API invokes: `kata-runtime direct-volume stats –-volume-path "/kubelet/a/b/c/d/sdf"`.
-2. The Kata runtime checks whether there is a sandbox id file under the directory `/run/kata-containers/shared/direct-volumes/kubelet/a/b/c/d/sdf`.
-3. The Kata runtime identifies the shim instance through the sandbox id, and sends a GRPC request to get the volume stats.
-4. The shim handles the request and forwards it to the Kata agent.
-5. Kata agent receives the request and returns the filesystem stats.

docs/design/host-cgroups.md

@@ -12,7 +12,7 @@ The OCI [runtime specification][linux-config] provides guidance on where the con
   > [`cgroupsPath`][cgroupspath]: (string, OPTIONAL) path to the cgroups. It can be used to either control the cgroups
   > hierarchy for containers or to run a new process in an existing container
 
-The cgroups are hierarchical, and this can be seen with the following pod example:
+Cgroups are hierarchical, and this can be seen with the following pod example:
 
 - Pod 1: `cgroupsPath=/kubepods/pod1`
   - Container 1: `cgroupsPath=/kubepods/pod1/container1`
@@ -247,14 +247,14 @@ cgroup size and constraints accordingly.
 
 # Supported cgroups
 
-Kata Containers currently supports cgroups `v1` and `v2`. 
+Kata Containers currently only supports cgroups `v1`. 
 
 In the following sections each cgroup is described briefly.
 
-## cgroups v1
+## Cgroups V1
 
-`cgroups v1` are under a [`tmpfs`][1] filesystem mounted at `/sys/fs/cgroup`, where each cgroup is
-mounted under a separate cgroup filesystem. A `cgroups v1` hierarchy may look like the following
+`Cgroups V1` are under a [`tmpfs`][1] filesystem mounted at `/sys/fs/cgroup`, where each cgroup is
+mounted under a separate cgroup filesystem. A `Cgroups v1` hierarchy may look like the following
 diagram:
 
 ```
@@ -301,12 +301,13 @@ diagram:
 A process can join a cgroup by writing its process id (`pid`) to `cgroup.procs` file,
 or join a cgroup partially by writing the task (thread) id (`tid`) to the `tasks` file.
 
+Kata Containers only supports `v1`.
 To know more about `cgroups v1`, see [cgroupsv1(7)][2].
 
-## cgroups v2
+## Cgroups V2
 
-`cgroups v2` are also known as unified cgroups, unlike `cgroups v1`, the cgroups are
-mounted under the same cgroup filesystem. A `cgroups v2` hierarchy may look like the following
+`Cgroups v2` are also known as unified cgroups, unlike `cgroups v1`, the cgroups are
+mounted under the same cgroup filesystem. A `Cgroups v2` hierarchy may look like the following
 diagram:
 
 ```
@@ -353,6 +354,8 @@ Same as `cgroups v1`, a process can join the cgroup by writing its process id (`
 `cgroup.procs` file, or join a cgroup partially by writing the task (thread) id (`tid`) to
 `cgroup.threads` file.
 
+Kata Containers does not support cgroups `v2` on the host.
+
 ### Distro Support
 
 Many Linux distributions do not yet support `cgroups v2`, as it is quite a recent addition.

docs/design/vcpu-handling.md

@@ -2,15 +2,24 @@
 
 ## Default number of virtual CPUs
 
-Before starting a container, the [runtime][4] reads the `default_vcpus` option
-from the [configuration file][5] to determine the number of virtual CPUs
+Before starting a container, the [runtime][6] reads the `default_vcpus` option
+from the [configuration file][7] to determine the number of virtual CPUs
 (vCPUs) needed to start the virtual machine. By default, `default_vcpus` is
 equal to 1 for fast boot time and a small memory footprint per virtual machine.
 Be aware that increasing this value negatively impacts the virtual machine's
 boot time and memory footprint.
 In general, we recommend that you do not edit this variable, unless you know
 what are you doing. If your container needs more than one vCPU, use
-[Kubernetes `cpu` limits][1] to assign more resources.
+[docker `--cpus`][1], [docker update][4], or [Kubernetes `cpu` limits][2] to
+assign more resources.
+
+*Docker*
+
+```sh
+$ docker run --name foo -ti --cpus 2 debian bash
+$ docker update --cpus 4 foo
+```
+
 
 *Kubernetes*
 
@@ -40,7 +49,7 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 ## Virtual CPUs and Kubernetes pods
 
 A Kubernetes pod is a group of one or more containers, with shared storage and
-network, and a specification for how to run the containers [[specification][2]].
+network, and a specification for how to run the containers [[specification][3]].
 In Kata Containers this group of containers, which is called a sandbox, runs inside
 the same virtual machine. If you do not specify a CPU constraint, the runtime does
 not add more vCPUs and the container is not placed inside a CPU cgroup.
@@ -64,7 +73,13 @@ constraints with each container trying to consume 100% of vCPU, the resources
 divide in two parts, 50% of vCPU for each container because your virtual
 machine does not have enough resources to satisfy containers needs. If you want
 to give access to a greater or lesser portion of vCPUs to a specific container,
-use [Kubernetes `cpu` requests][1].
+use [`docker --cpu-shares`][1] or [Kubernetes `cpu` requests][2].
+
+*Docker*
+
+```sh
+$ docker run -ti --cpus-shares=512 debian bash
+```
 
 *Kubernetes*
 
@@ -94,9 +109,10 @@ $ sudo -E kubectl create -f ~/cpu-demo.yaml
 Before running containers without CPU constraint, consider that your containers
 are not running alone. Since your containers run inside a virtual machine other
 processes use the vCPUs as well (e.g. `systemd` and the Kata Containers
-[agent][3]). In general, we recommend setting `default_vcpus` equal to 1 to
+[agent][5]). In general, we recommend setting `default_vcpus` equal to 1 to
 allow non-container processes to run on this vCPU and to specify a CPU
-constraint for each container.
+constraint for each container. If your container is already running and needs
+more vCPUs, you can add more using [docker update][4].
 
 ## Container with CPU constraint
 
@@ -105,7 +121,7 @@ constraints using the following formula: `vCPUs = ceiling( quota / period )`, wh
 `quota` specifies the number of microseconds per CPU Period that the container is
 guaranteed CPU access and `period` specifies the CPU CFS scheduler period of time
 in microseconds. The result determines the number of vCPU to hot plug into the
-virtual machine. Once the vCPUs have been added, the [agent][3] places the
+virtual machine. Once the vCPUs have been added, the [agent][5] places the
 container inside a CPU cgroup. This placement allows the container to use only
 its assigned resources.
 
@@ -122,6 +138,25 @@ the virtual machine starts with 8 vCPUs and 1 vCPUs is added and assigned
 to the container. Non-container processes might be able to use 8 vCPUs but they
 use a maximum 1 vCPU, hence 7 vCPUs might not be used.
 
+
+*Container without CPU constraint*
+
+```sh
+$ docker run -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
+1       # number of vCPUs
+100000  # cfs period
+-1      # cfs quota
+```
+
+*Container with CPU constraint*
+
+```sh
+docker run --cpus 4 -ti debian bash -c "nproc; cat /sys/fs/cgroup/cpu,cpuacct/cpu.cfs_*"
+5       # number of vCPUs
+100000  # cfs period
+400000  # cfs quota
+```
+
 ## Virtual CPU handling without hotplug
 
 In some cases, the hardware and/or software architecture being utilized does not support
@@ -148,8 +183,11 @@ the container's `spec` will provide the sizing information directly. If these ar
 calculate the number of CPUs required for the workload and augment this by `default_vcpus`
 configuration option, and use this for the virtual machine size.
 
-[1]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
-[2]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
-[3]: ../../src/agent
-[4]: ../../src/runtime
-[5]: ../../src/runtime/README.md#configuration
+
+[1]: https://docs.docker.com/config/containers/resource_constraints/#cpu
+[2]: https://kubernetes.io/docs/tasks/configure-pod-container/assign-cpu-resource
+[3]: https://kubernetes.io/docs/concepts/workloads/pods/pod/
+[4]: https://docs.docker.com/engine/reference/commandline/update/
+[5]: ../../src/agent
+[6]: ../../src/runtime
+[7]: ../../src/runtime/README.md#configuration

docs/design/virtualization.md

@@ -39,7 +39,7 @@ Details of each solution and a summary are provided below.
 Kata Containers with QEMU has complete compatibility with Kubernetes.
 
 Depending on the host architecture, Kata Containers supports various machine types,
-for example `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
+for example `pc` and `q35` on x86 systems, `virt` on ARM systems and `pseries` on IBM Power systems. The default Kata Containers
 machine type is `q35`. The machine type and its [`Machine accelerators`](#machine-accelerators) can
 be changed by editing the runtime [`configuration`](architecture/README.md#configuration) file.
 
@@ -60,8 +60,9 @@ Machine accelerators are architecture specific and can be used to improve the pe
 and enable specific features of the machine types. The following machine accelerators
 are used in Kata Containers:
 
-- NVDIMM: This machine accelerator is x86 specific and only supported by `q35` machine types.
-`nvdimm` is used to provide the root filesystem as a persistent memory device to the Virtual Machine.
+- NVDIMM: This machine accelerator is x86 specific and only supported by `pc` and
+`q35` machine types. `nvdimm` is used to provide the root filesystem as a persistent
+memory device to the Virtual Machine.
 
 #### Hotplug devices
 
@@ -110,7 +111,7 @@ Devices and features used:
 - VFIO
 - hotplug
 - seccomp filters
-- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/main/vmm/src/api/openapi/cloud-hypervisor.yaml)
+- [HTTP OpenAPI](https://github.com/cloud-hypervisor/cloud-hypervisor/blob/master/vmm/src/api/openapi/cloud-hypervisor.yaml)
 
 ### Summary
 

docs/how-to/README.md

@@ -5,7 +5,7 @@
 - [Run Kata containers with `crictl`](run-kata-with-crictl.md)
 - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 - [How to use Kata Containers and Containerd](containerd-kata.md)
-- [How to use Kata Containers and containerd with Kubernetes](how-to-use-k8s-with-containerd-and-kata.md)
+- [How to use Kata Containers and CRI (containerd) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
 - [Kata Containers and service mesh for Kubernetes](service-mesh.md)
 - [How to import Kata Containers logs into Fluentd](how-to-import-kata-logs-with-fluentd.md)
 
@@ -15,11 +15,6 @@
 - `qemu`
 - `cloud-hypervisor`
 - `firecracker`
-
-   In the case of `firecracker` the use of a block device `snapshotter` is needed
-   for the VM rootfs. Refer to the following guide for additional configuration
-   steps:
-   - [Setup Kata containers with `firecracker`](how-to-use-kata-containers-with-firecracker.md)
 - `ACRN`
 
   While `qemu` , `cloud-hypervisor` and `firecracker` work out of the box with installation of Kata,
@@ -42,9 +37,4 @@
 - [How to setup swap devices in guest kernel](how-to-setup-swap-devices-in-guest-kernel.md)
 - [How to run rootless vmm](how-to-run-rootless-vmm.md)
 - [How to run Docker with Kata Containers](how-to-run-docker-with-kata.md)
-- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)
-- [How to run Kata Containers with AMD SEV-SNP](how-to-run-kata-containers-with-SNP-VMs.md)
-
-## Confidential Containers
-- [How to use build and test the Confidential Containers `CCv0` proof of concept](how-to-build-and-test-ccv0.md)
-- [How to generate a Kata Containers payload for the Confidential Containers Operator](how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md)
+- [How to run Kata Containers with `nydus`](how-to-use-virtio-fs-nydus-with-kata.md)

docs/how-to/ccv0.sh

@@ -1,640 +0,0 @@
-#!/bin/bash -e
-#
-# Copyright (c) 2021, 2022 IBM Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-# Disclaimer: This script is work in progress for supporting the CCv0 prototype
-# It shouldn't be considered supported by the Kata Containers community, or anyone else
-
-# Based on https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md,
-# but with elements of the tests/.ci scripts used
-
-readonly script_name="$(basename "${BASH_SOURCE[0]}")"
-
-# By default in Golang >= 1.16 GO111MODULE is set to "on", but not all modules support it, so overwrite to "auto"
-export GO111MODULE="auto"
-
-# Setup kata containers environments if not set - we default to use containerd
-export CRI_CONTAINERD=${CRI_CONTAINERD:-"yes"}
-export CRI_RUNTIME=${CRI_RUNTIME:-"containerd"}
-export CRIO=${CRIO:-"no"}
-export KATA_HYPERVISOR="${KATA_HYPERVISOR:-qemu}"
-export KUBERNETES=${KUBERNETES:-"no"}
-export AGENT_INIT="${AGENT_INIT:-${TEST_INITRD:-no}}"
-export AA_KBC="${AA_KBC:-offline_fs_kbc}"
-
-# Allow the user to overwrite the default repo and branch names if they want to build from a fork
-export katacontainers_repo="${katacontainers_repo:-github.com/kata-containers/kata-containers}"
-export katacontainers_branch="${katacontainers_branch:-CCv0}" 
-export kata_default_branch=${katacontainers_branch}
-export tests_repo="${tests_repo:-github.com/kata-containers/tests}"
-export tests_branch="${tests_branch:-CCv0}"
-export target_branch=${tests_branch} # kata-containers/ci/lib.sh uses target branch var to check out tests repo
-
-# if .bash_profile exists then use it, otherwise fall back to .profile
-export PROFILE="${HOME}/.profile"
-if [ -r "${HOME}/.bash_profile" ]; then
-    export PROFILE="${HOME}/.bash_profile"
-fi
-# Stop PS1: unbound variable error happening
-export PS1=${PS1:-}
-
-# Create a bunch of common, derived values up front so we don't need to create them in all the different functions
-. ${PROFILE}
-if [ -z ${GOPATH} ]; then
-    export GOPATH=${HOME}/go
-fi
-export tests_repo_dir="${GOPATH}/src/${tests_repo}"
-export katacontainers_repo_dir="${GOPATH}/src/${katacontainers_repo}"
-export ROOTFS_DIR="${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder/rootfs"
-export PULL_IMAGE="${PULL_IMAGE:-quay.io/kata-containers/confidential-containers:signed}" # Doesn't need authentication
-export CONTAINER_ID="${CONTAINER_ID:-0123456789}"
-source /etc/os-release || source /usr/lib/os-release
-grep -Eq "\<fedora\>" /etc/os-release 2> /dev/null && export USE_PODMAN=true
-
-
-# If we've already checked out the test repo then source the confidential scripts
-if [ "${KUBERNETES}" == "yes" ]; then
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/kubernetes/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-else
-    export BATS_TEST_DIRNAME="${tests_repo_dir}/integration/containerd/confidential"
-    [ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/lib.sh"
-fi
-
-[ -d "${BATS_TEST_DIRNAME}" ] && source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-
-export RUNTIME_CONFIG_PATH=/etc/kata-containers/configuration.toml
-
-usage() {
-    exit_code="$1"
-    cat <<EOF
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/${script_name} build_and_install_all
-Usage:
-    ${script_name} [options] <command>
-Commands:
-- agent_create_container:           Run CreateContainer command against the agent with agent-ctl
-- agent_pull_image:                 Run PullImage command against the agent with agent-ctl
-- all:                              Build and install everything, test kata with containerd and capture the logs
-- build_and_add_agent_to_rootfs:    Builds the kata-agent and adds it to the rootfs
-- build_and_install_all:            Build and install everything
-- build_and_install_rootfs:         Builds and installs the rootfs image
-- build_kata_runtime:               Build and install the kata runtime
-- build_cloud_hypervisor            Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                       Checkout, patch, build and install QEMU
-- configure:                        Configure Kata to use rootfs and enable debug
-- connect_to_ssh_demo_pod:          Ssh into the ssh demo pod, showing that the decryption succeeded
-- copy_signature_files_to_guest     Copies signature verification files to guest
-- create_rootfs:                    Create a local rootfs
-- crictl_create_cc_container        Use crictl to create a new busybox container in the kata cc pod
-- crictl_create_cc_pod              Use crictl to create a new kata cc pod
-- crictl_delete_cc                  Use crictl to delete the kata cc pod sandbox and container in it
-- help:                             Display this help
-- init_kubernetes:                  initialize a Kubernetes cluster on this system
-- initialize:                       Install dependencies and check out kata-containers source
-- install_guest_kernel:             Setup, build and install the guest kernel
-- kubernetes_create_cc_pod:         Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_create_ssh_demo_pod:   Create a Kata CC runtime pod based on the ssh demo
-- kubernetes_delete_cc_pod:         Delete the Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_ssh_demo_pod:   Delete the Kata CC runtime pod based on the ssh demo
-- open_kata_shell:                  Open a shell into the kata runtime
-- rebuild_and_install_kata:         Rebuild the kata runtime and agent and build and install the image
-- shim_pull_image:                  Run PullImage command against the shim with ctr
-- test_capture_logs:                Test using kata with containerd and capture the logs in the user's home directory
-- test:                             Test using kata with containerd
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-EOF
-    # if script sourced don't exit as this will exit the main shell, just return instead
-    [[ $_ != $0 ]] && return "$exit_code" || exit "$exit_code"
-}
-
-build_and_install_all() {
-    initialize
-    build_and_install_kata_runtime
-    configure
-    create_a_local_rootfs
-    build_and_install_rootfs
-    install_guest_kernel_image
-    case "$KATA_HYPERVISOR" in
-        "qemu") 
-            build_qemu
-            ;;
-        "cloud-hypervisor") 
-            build_cloud_hypervisor
-            ;;
-        *)
-            echo "Invalid option: $KATA_HYPERVISOR is not supported." >&2
-            ;;
-    esac
-
-    check_kata_runtime
-    if [ "${KUBERNETES}" == "yes" ]; then
-        init_kubernetes
-    fi
-}
-
-rebuild_and_install_kata() {
-    checkout_tests_repo
-    checkout_kata_containers_repo
-    build_and_install_kata_runtime
-    build_and_add_agent_to_rootfs
-    build_and_install_rootfs
-    check_kata_runtime
-}
-
-# Based on the jenkins_job_build.sh script in kata-containers/tests/.ci - checks out source code and installs dependencies
-initialize() {
-    # We need git to checkout and bootstrap the ci scripts and some other packages used in testing
-    sudo apt-get update && sudo apt-get install -y curl git qemu-utils
-    
-    grep -qxF "export GOPATH=\${HOME}/go" "${PROFILE}" || echo "export GOPATH=\${HOME}/go" >> "${PROFILE}"
-    grep -qxF "export GOROOT=/usr/local/go" "${PROFILE}" || echo "export GOROOT=/usr/local/go" >> "${PROFILE}"
-    grep -qxF "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" "${PROFILE}" || echo "export PATH=\${GOPATH}/bin:/usr/local/go/bin:\${PATH}" >> "${PROFILE}"
-    
-    # Load the new go and PATH parameters from the profile
-    . ${PROFILE}
-    mkdir -p "${GOPATH}"
-
-    checkout_tests_repo
-
-    pushd "${tests_repo_dir}"
-    local ci_dir_name=".ci"
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_go.sh" -p -f
-    sudo -E PATH=$PATH -s "${ci_dir_name}/install_rust.sh"
-    # Need to change ownership of rustup so later process can create temp files there
-    sudo chown -R ${USER}:${USER} "${HOME}/.rustup"
-
-    checkout_kata_containers_repo
-
-    # Run setup, but don't install kata as we will build it ourselves in locations matching the developer guide
-    export INSTALL_KATA="no"
-    sudo -E PATH=$PATH -s ${ci_dir_name}/setup.sh
-    # Reload the profile to pick up installed dependencies
-    . ${PROFILE}
-    popd
-}
-
-checkout_tests_repo() {
-    echo "Creating repo: ${tests_repo} and branch ${tests_branch} into ${tests_repo_dir}..."
-    # Due to git https://github.blog/2022-04-12-git-security-vulnerability-announced/ the tests repo needs
-    # to be owned by root as it is re-checked out in rootfs.sh
-    mkdir -p $(dirname "${tests_repo_dir}")
-    [ -d "${tests_repo_dir}" ] || sudo -E git clone "https://${tests_repo}.git" "${tests_repo_dir}"
-    sudo -E chown -R root:root "${tests_repo_dir}"
-    pushd "${tests_repo_dir}"
-    sudo -E git fetch
-    if [ -n "${tests_branch}" ]; then
-        sudo -E git checkout ${tests_branch}
-    fi
-    sudo -E git reset --hard origin/${tests_branch}
-    popd
-
-    source "${BATS_TEST_DIRNAME}/lib.sh"
-    source "${BATS_TEST_DIRNAME}/../../confidential/lib.sh"
-}
-
-# Note: clone_katacontainers_repo using go, so that needs to be installed first
-checkout_kata_containers_repo() {
-    source "${tests_repo_dir}/.ci/lib.sh"
-    echo "Creating repo: ${katacontainers_repo} and branch ${kata_default_branch} into ${katacontainers_repo_dir}..."
-    clone_katacontainers_repo
-    sudo -E chown -R ${USER}:${USER} "${katacontainers_repo_dir}"
-}
-
-build_and_install_kata_runtime() {
-    pushd ${katacontainers_repo_dir}/src/runtime
-    make clean && make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} && sudo -E PATH=$PATH make DEFAULT_HYPERVISOR=${KATA_HYPERVISOR} install
-    popd
-}
-
-configure() {
-    configure_kata_to_use_rootfs
-    enable_full_debug
-    enable_agent_console
-
-    # Switch image offload to true in kata config
-    switch_image_service_offload "on"
-
-    configure_cc_containerd
-    # From crictl v1.24.1 the default timoout leads to the pod creation failing, so update it
-    sudo crictl config --set timeout=10
-}
-
-configure_kata_to_use_rootfs() {
-    sudo mkdir -p /etc/kata-containers/
-    sudo install -o root -g root -m 0640 /usr/share/defaults/kata-containers/configuration.toml /etc/kata-containers
-    sudo sed -i 's/^\(initrd =.*\)/# \1/g' ${RUNTIME_CONFIG_PATH}
-}
-
-build_and_add_agent_to_rootfs() {
-    build_a_custom_kata_agent
-    add_custom_agent_to_rootfs
-}
-
-build_a_custom_kata_agent() {
-    # Install libseccomp for static linking
-    sudo -E PATH=$PATH GOPATH=$GOPATH ${katacontainers_repo_dir}/ci/install_libseccomp.sh /tmp/kata-libseccomp /tmp/kata-gperf
-    export LIBSECCOMP_LINK_TYPE=static
-    export LIBSECCOMP_LIB_PATH=/tmp/kata-libseccomp/lib
-
-    . "$HOME/.cargo/env"
-    pushd ${katacontainers_repo_dir}/src/agent
-    sudo -E PATH=$PATH make
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    # Run a make install into the rootfs directory in order to create the kata-agent.service file which is required when we add to the rootfs
-    sudo -E PATH=$PATH make install DESTDIR="${ROOTFS_DIR}"
-    popd
-}
-
-create_a_local_rootfs() {
-    sudo rm -rf "${ROOTFS_DIR}"
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-    export distro="ubuntu"
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E OS_VERSION="${OS_VERSION:-}" GOPATH=$GOPATH EXTRA_PKGS="vim iputils-ping net-tools" DEBUG="${DEBUG:-}" USE_DOCKER="${use_docker:-}" SKOPEO=${SKOPEO:-} AA_KBC=${AA_KBC:-} UMOCI=yes SECCOMP=yes ./rootfs.sh -r ${ROOTFS_DIR} ${distro}
-
-     # Install_rust.sh during rootfs.sh switches us to the main branch of the tests repo, so switch back now
-    pushd "${tests_repo_dir}"
-    sudo -E git checkout ${tests_branch}
-    popd
-    # During the ./rootfs.sh call the kata agent is built as root, so we need to update the permissions, so we can rebuild it
-    sudo chown -R ${USER}:${USER} "${katacontainers_repo_dir}/src/agent/"
-
-    popd
-}
-
-add_custom_agent_to_rootfs() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/rootfs-builder
-
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-
-    sudo install -o root -g root -m 0550 -t ${ROOTFS_DIR}/usr/bin ${katacontainers_repo_dir}/src/agent/target/${ARCH}-unknown-linux-${LIBC}/release/kata-agent
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-agent.service ${ROOTFS_DIR}/usr/lib/systemd/system/
-    sudo install -o root -g root -m 0440 ../../../src/agent/kata-containers.target ${ROOTFS_DIR}/usr/lib/systemd/system/
-    popd
-}
-
-build_and_install_rootfs() {
-    build_rootfs_image
-    install_rootfs_image
-}
-
-build_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    # Logic from install_kata_image.sh - if we aren't using podman (ie on a fedora like), then use docker
-    [[ -z "${USE_PODMAN:-}" ]] && use_docker="${use_docker:-1}"
-    sudo -E USE_DOCKER="${use_docker:-}" ./image_builder.sh ${ROOTFS_DIR}
-    popd
-}
-
-install_rootfs_image() {
-    pushd ${katacontainers_repo_dir}/tools/osbuilder/image-builder
-    local commit=$(git log --format=%h -1 HEAD)
-    local date=$(date +%Y-%m-%d-%T.%N%z)
-    local image="kata-containers-${date}-${commit}"
-    sudo install -o root -g root -m 0640 -D kata-containers.img "/usr/share/kata-containers/${image}"
-    (cd /usr/share/kata-containers && sudo ln -sf "$image" kata-containers.img)
-    echo "Built Rootfs from ${ROOTFS_DIR} to /usr/share/kata-containers/${image}"
-    ls -al /usr/share/kata-containers/
-    popd
-}
-
-install_guest_kernel_image() {
-    pushd ${katacontainers_repo_dir}/tools/packaging/kernel
-    sudo -E PATH=$PATH ./build-kernel.sh setup
-    sudo -E PATH=$PATH ./build-kernel.sh build
-    sudo chmod u+wrx /usr/share/kata-containers/ # Give user permission to install kernel
-    sudo -E PATH=$PATH ./build-kernel.sh install
-    popd
-}
-
-build_qemu() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_qemu.sh
-}
-
-build_cloud_hypervisor() {
-    ${tests_repo_dir}/.ci/install_virtiofsd.sh
-    ${tests_repo_dir}/.ci/install_cloud_hypervisor.sh
-}
-
-check_kata_runtime() {
-    sudo kata-runtime check
-}
-
-k8s_pod_file="${HOME}/busybox-cc.yaml"
-init_kubernetes() {
-    # Check that kubeadm was installed and install it otherwise
-    if ! [ -x "$(command -v kubeadm)" ]; then
-        pushd "${tests_repo_dir}/.ci"
-        sudo -E PATH=$PATH -s install_kubernetes.sh
-        if [ "${CRI_CONTAINERD}" == "yes" ]; then
-            sudo -E PATH=$PATH -s "configure_containerd_for_kubernetes.sh"
-        fi
-        popd
-    fi
-
-    # If kubernetes init has previously run we need to clean it by removing the image and resetting k8s
-    local cid=$(sudo docker ps -a -q -f name=^/kata-registry$)
-    if [ -n "${cid}" ]; then
-        sudo docker stop ${cid} && sudo docker rm ${cid}
-    fi
-    local k8s_nodes=$(kubectl get nodes -o name 2>/dev/null || true)
-    if [ -n "${k8s_nodes}" ]; then
-        sudo kubeadm reset -f
-    fi
-
-    export CI="true" && sudo -E PATH=$PATH -s ${tests_repo_dir}/integration/kubernetes/init.sh
-    sudo chown ${USER}:$(id -g -n ${USER}) "$HOME/.kube/config"
-    cat << EOF > ${k8s_pod_file}
-apiVersion: v1
-kind: Pod
-metadata:
-  name: busybox-cc
-spec:
-  runtimeClassName: kata
-  containers:
-  - name: nginx
-    image: quay.io/kata-containers/confidential-containers:signed
-    imagePullPolicy: Always  
-EOF
-}
-
-call_kubernetes_create_cc_pod() {
-    kubernetes_create_cc_pod ${k8s_pod_file}
-}
-
-call_kubernetes_delete_cc_pod() {
-    pod_name=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_cc_pod $pod_name
-}
-
-call_kubernetes_create_ssh_demo_pod() {
-    setup_decryption_files_in_guest
-    kubernetes_create_ssh_demo_pod
-}
-
-call_connect_to_ssh_demo_pod() {
-    connect_to_ssh_demo_pod
-}
-
-call_kubernetes_delete_ssh_demo_pod() {
-    pod=$(kubectl get pods -o jsonpath='{.items..metadata.name}')
-    kubernetes_delete_ssh_demo_pod $pod
-}
-
-crictl_sandbox_name=kata-cc-busybox-sandbox
-call_crictl_create_cc_pod() {
-    # Update iptables to allow forwarding to the cni0 bridge avoiding issues caused by the docker0 bridge
-    sudo iptables -P FORWARD ACCEPT
-    
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    crictl_delete_cc_pod_if_exists "${crictl_sandbox_name}"
-    crictl_create_cc_pod "${pod_config}"
-    sudo crictl pods
-}
-
-call_crictl_create_cc_container() {
-    # Create container configuration yaml based on our test copy of busybox
-    # get_pod_config in tests_common exports `pod_config` that points to the prepared pod config yaml 
-    get_pod_config
-
-    local container_config="${FIXTURES_DIR}/${CONTAINER_CONFIG_FILE:-container-config.yaml}"
-    local pod_name=${crictl_sandbox_name}
-    crictl_create_cc_container ${pod_name} ${pod_config} ${container_config}
-    sudo crictl ps -a
-}
-
-crictl_delete_cc() {
-    crictl_delete_cc_pod ${crictl_sandbox_name}
-}
-
-test_kata_runtime() {
-    echo "Running ctr with the kata runtime..."
-    local test_image="quay.io/kata-containers/confidential-containers:signed"
-    if [ -z $(ctr images ls -q name=="${test_image}") ]; then
-        sudo ctr image pull "${test_image}"
-    fi
-    sudo ctr run --runtime "io.containerd.kata.v2" --rm -t "${test_image}" test-kata uname -a
-}
-
-run_kata_and_capture_logs() {
-    echo "Clearing systemd journal..."
-    sudo systemctl stop systemd-journald
-    sudo rm -f /var/log/journal/*/* /run/log/journal/*/*
-    sudo systemctl start systemd-journald
-    test_kata_runtime
-    echo "Collecting logs..."
-    sudo journalctl -q -o cat -a -t kata-runtime > ${HOME}/kata-runtime.log
-    sudo journalctl -q -o cat -a -t kata > ${HOME}/shimv2.log
-    echo "Logs output to ${HOME}/kata-runtime.log and ${HOME}/shimv2.log"
-}
-
-get_ids() {
-    guest_cid=$(sudo ss -H --vsock | awk '{print $6}' | cut -d: -f1)
-    sandbox_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-}
-
-open_kata_shell() {
-    get_ids
-    sudo -E "PATH=$PATH" kata-runtime exec ${sandbox_id}
-}
-
-build_bundle_dir_if_necessary() {
-    bundle_dir="/tmp/bundle"
-    if [ ! -d "${bundle_dir}" ]; then
-        rootfs_dir="$bundle_dir/rootfs"
-        image="quay.io/kata-containers/confidential-containers:signed"
-        mkdir -p "$rootfs_dir" && (cd "$bundle_dir" && runc spec)
-        sudo docker export $(sudo docker create "$image") | tar -C "$rootfs_dir" -xvf -
-    fi
-    # There were errors in create container agent-ctl command due to /bin/ seemingly not being on the path, so hardcode it
-    sudo sed -i -e 's%^\(\t*\)"sh"$%\1"/bin/sh"%g' "${bundle_dir}/config.json"
-}
-
-build_agent_ctl() {
-    cd ${GOPATH}/src/${katacontainers_repo}/src/tools/agent-ctl/
-    if [ -e "${HOME}/.cargo/registry" ]; then
-        sudo chown -R ${USER}:${USER} "${HOME}/.cargo/registry"
-    fi
-    sudo -E PATH=$PATH -s  make
-    ARCH=$(uname -m)
-    [ ${ARCH} == "ppc64le" ] || [ ${ARCH} == "s390x" ] && export LIBC=gnu || export LIBC=musl
-    [ ${ARCH} == "ppc64le" ] && export ARCH=powerpc64le
-    cd "./target/${ARCH}-unknown-linux-${LIBC}/release/"
-}
-
-run_agent_ctl_command() {
-    get_ids
-    build_bundle_dir_if_necessary
-    command=$1
-    # If kata-agent-ctl pre-built in this directory, use it directly, otherwise build it first and switch to release
-    if [ ! -x kata-agent-ctl ]; then
-        build_agent_ctl
-    fi 
-     ./kata-agent-ctl -l debug connect --bundle-dir "${bundle_dir}" --server-address "vsock://${guest_cid}:1024" -c "${command}"
-}
-
-agent_pull_image() {
-    run_agent_ctl_command "PullImage image=${PULL_IMAGE} cid=${CONTAINER_ID} source_creds=${SOURCE_CREDS}"
-}
-
-agent_create_container() {
-    run_agent_ctl_command "CreateContainer cid=${CONTAINER_ID}"
-}
-
-shim_pull_image() {
-    get_ids
-    local ctr_shim_command="sudo ctr --namespace k8s.io shim --id ${sandbox_id} pull-image ${PULL_IMAGE} ${CONTAINER_ID}"
-    echo "Issuing command '${ctr_shim_command}'"
-    ${ctr_shim_command}
-}
-
-call_copy_signature_files_to_guest() {
-    # TODO #5173 - remove this once the kernel_params aren't ignored by the agent config
-    export DEBUG_CONSOLE="true"
-    
-    if [ "${SKOPEO:-}" = "yes" ]; then
-        add_kernel_params "agent.container_policy_file=/etc/containers/quay_verification/quay_policy.json"
-        setup_skopeo_signature_files_in_guest
-    else
-        # TODO #4888 - set config to specifically enable signature verification to be on in ImageClient
-        setup_offline_fs_kbc_signature_files_in_guest
-    fi
-}
-
-main() {
-    while getopts "dh" opt; do
-        case "$opt" in
-            d) 
-                export DEBUG="-d"
-                set -x
-                ;;
-            h) 
-                usage 0
-                ;;
-            \?)
-                echo "Invalid option: -$OPTARG" >&2
-                usage 1
-                ;;
-        esac
-    done
-
-    shift $((OPTIND - 1))
-
-    subcmd="${1:-}"
-
-    [ -z "${subcmd}" ] && usage 1
-
-    case "${subcmd}" in
-        all)
-            build_and_install_all
-            run_kata_and_capture_logs
-            ;;
-        build_and_install_all)
-            build_and_install_all
-            ;;
-        rebuild_and_install_kata)
-            rebuild_and_install_kata
-            ;;
-        initialize)
-            initialize
-            ;;
-        build_kata_runtime)
-            build_and_install_kata_runtime
-            ;;
-        configure)
-            configure
-            ;;
-        create_rootfs)
-            create_a_local_rootfs
-            ;;
-        build_and_add_agent_to_rootfs)
-            build_and_add_agent_to_rootfs
-            ;;
-        build_and_install_rootfs)
-            build_and_install_rootfs
-            ;;
-        install_guest_kernel)
-            install_guest_kernel_image
-            ;;
-        build_cloud_hypervisor)
-            build_cloud_hypervisor
-            ;;
-        build_qemu)
-            build_qemu
-            ;;
-        init_kubernetes)
-            init_kubernetes
-            ;;
-        crictl_create_cc_pod)
-            call_crictl_create_cc_pod
-            ;;
-        crictl_create_cc_container)
-            call_crictl_create_cc_container
-            ;;
-        crictl_delete_cc)
-            crictl_delete_cc
-            ;;
-        kubernetes_create_cc_pod)
-            call_kubernetes_create_cc_pod
-            ;;
-        kubernetes_delete_cc_pod)
-            call_kubernetes_delete_cc_pod
-            ;;
-        kubernetes_create_ssh_demo_pod)
-            call_kubernetes_create_ssh_demo_pod
-            ;;
-        connect_to_ssh_demo_pod)
-            call_connect_to_ssh_demo_pod
-            ;;
-        kubernetes_delete_ssh_demo_pod)
-            call_kubernetes_delete_ssh_demo_pod
-            ;;
-        test)
-            test_kata_runtime
-            ;;
-        test_capture_logs)
-            run_kata_and_capture_logs
-            ;;
-        open_kata_console)
-            open_kata_console
-            ;;
-        open_kata_shell)
-            open_kata_shell
-            ;;
-        agent_pull_image)
-            agent_pull_image
-            ;;
-        shim_pull_image)
-            shim_pull_image
-            ;;
-        agent_create_container)
-            agent_create_container
-            ;;
-        copy_signature_files_to_guest)
-            call_copy_signature_files_to_guest
-            ;;
-        *)
-            usage 1
-            ;;
-    esac
-}
-
-main $@

docs/how-to/containerd-kata.md

@@ -40,7 +40,7 @@ use `RuntimeClass` instead of the deprecated annotations.
 ### Containerd Runtime V2 API: Shim V2 API
 
 The [`containerd-shim-kata-v2` (short as `shimv2` in this documentation)](../../src/runtime/cmd/containerd-shim-kata-v2/)
-implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/runtime/v2) for Kata.
+implements the [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2) for Kata.
 With `shimv2`, Kubernetes can launch Pod and OCI-compatible containers with one shim per Pod. Prior to `shimv2`, `2N+1` 
 shims (i.e. a `containerd-shim` and a `kata-shim` for each container and the Pod sandbox itself) and no standalone `kata-proxy` 
 process were used, even with VSOCK not available.
@@ -72,13 +72,14 @@ $ command -v containerd
 
 ### Install CNI plugins
 
+> **Note:** You do not need to install CNI plugins if you do not want to use containerd with Kubernetes.
 > If you have installed Kubernetes with `kubeadm`, you might have already installed the CNI plugins.
 
 You can manually install CNI plugins as follows:
 
 ```bash
-$ git clone https://github.com/containernetworking/plugins.git
-$ pushd plugins
+$ go get github.com/containernetworking/plugins
+$ pushd $GOPATH/src/github.com/containernetworking/plugins
 $ ./build_linux.sh
 $ sudo mkdir /opt/cni
 $ sudo cp -r bin /opt/cni/
@@ -93,8 +94,8 @@ $ popd
 You can install the `cri-tools` from source code:
 
 ```bash
-$ git clone https://github.com/kubernetes-sigs/cri-tools.git
-$ pushd cri-tools
+$ go get github.com/kubernetes-incubator/cri-tools
+$ pushd $GOPATH/src/github.com/kubernetes-incubator/cri-tools
 $ make
 $ sudo -E make install
 $ popd
@@ -130,42 +131,74 @@ For
 
 The `RuntimeClass` is suggested.
 
-The following configuration includes two runtime classes:
+The following configuration includes three runtime classes:
 - `plugins.cri.containerd.runtimes.runc`: the runc, and it is the default runtime.
-- `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/main/runtime/v2#binary-naming)) 
+- `plugins.cri.containerd.runtimes.kata`: The function in containerd (reference [the document here](https://github.com/containerd/containerd/tree/master/runtime/v2#binary-naming)) 
   where the dot-connected string `io.containerd.kata.v2` is translated to `containerd-shim-kata-v2` (i.e. the 
-  binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/main/runtime/v2)).
+  binary name of the Kata implementation of [Containerd Runtime V2 (Shim API)](https://github.com/containerd/containerd/tree/master/runtime/v2)).
+- `plugins.cri.containerd.runtimes.katacli`: the `containerd-shim-runc-v1` calls `kata-runtime`, which is the legacy process.
 
 ```toml
     [plugins.cri.containerd]
       no_pivot = false
     [plugins.cri.containerd.runtimes]
-      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
-         privileged_without_host_devices = false
-         runtime_type = "io.containerd.runc.v2"
-        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
-            BinaryName = ""
-            CriuImagePath = ""
-            CriuPath = ""
-            CriuWorkPath = ""
-            IoGid = 0
+      [plugins.cri.containerd.runtimes.runc]
+         runtime_type = "io.containerd.runc.v1"
+         [plugins.cri.containerd.runtimes.runc.options]
+           NoPivotRoot = false
+           NoNewKeyring = false
+           ShimCgroup = ""
+           IoUid = 0
+           IoGid = 0
+           BinaryName = "runc"
+           Root = ""
+           CriuPath = ""
+           SystemdCgroup = false
       [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"
-         privileged_without_host_devices = true
-         pod_annotations = ["io.katacontainers.*"]
-         container_annotations = ["io.katacontainers.*"]
-         [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata.options]
-            ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration.toml"
+      [plugins.cri.containerd.runtimes.katacli]
+         runtime_type = "io.containerd.runc.v1"
+         [plugins.cri.containerd.runtimes.katacli.options]
+           NoPivotRoot = false
+           NoNewKeyring = false
+           ShimCgroup = ""
+           IoUid = 0
+           IoGid = 0
+           BinaryName = "/usr/bin/kata-runtime"
+           Root = ""
+           CriuPath = ""
+           SystemdCgroup = false
+```
+
+From Containerd v1.2.4 and Kata v1.6.0, there is a new runtime option supported, which allows you to specify a specific Kata configuration file as follows:
+
+```toml
+      [plugins.cri.containerd.runtimes.kata]
+         runtime_type = "io.containerd.kata.v2"
+	 privileged_without_host_devices = true
+	 [plugins.cri.containerd.runtimes.kata.options]
+	   ConfigPath = "/etc/kata-containers/config.toml"
 ```
 
 `privileged_without_host_devices` tells containerd that a privileged Kata container should not have direct access to all host devices. If unset, containerd will pass all host devices to Kata container, which may cause security issues.
 
-`pod_annotations` is the list of pod annotations passed to both the pod sandbox as well as container through the OCI config.
-
-`container_annotations` is the list of container annotations passed through to the OCI config of the containers.
-
 This `ConfigPath` option is optional. If you do not specify it, shimv2 first tries to get the configuration file from the environment variable `KATA_CONF_FILE`. If neither are set, shimv2 will use the default Kata configuration file paths (`/etc/kata-containers/configuration.toml` and `/usr/share/defaults/kata-containers/configuration.toml`).
 
+If you use Containerd older than v1.2.4 or a version of Kata older than v1.6.0  and also want to specify a configuration file, you can use the following workaround, since the shimv2 accepts an environment variable, `KATA_CONF_FILE` for the configuration file path. Then, you can create a
+shell script with the following:
+
+```bash
+#!/usr/bin/env bash
+KATA_CONF_FILE=/etc/kata-containers/firecracker.toml containerd-shim-kata-v2 $@
+```
+
+Name it as `/usr/local/bin/containerd-shim-katafc-v2` and reference it in the configuration of containerd:
+
+```toml
+      [plugins.cri.containerd.runtimes.kata-firecracker]
+         runtime_type = "io.containerd.katafc.v2"
+```
+
 #### Kata Containers as the runtime for untrusted workload
 
 For cases without `RuntimeClass` support, we can use the legacy annotation method to support using Kata Containers 
@@ -185,8 +218,28 @@ and then, run an untrusted workload with Kata Containers:
       runtime_type = "io.containerd.kata.v2"
 ```
 
+For the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
+
+```toml
+    [plugins.cri.containerd]
+  
+    # "plugins.cri.containerd.default_runtime" is the runtime to use in containerd.
+    [plugins.cri.containerd.default_runtime]
+      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
+      runtime_type = "io.containerd.runtime.v1.linux"
+
+    # "plugins.cri.containerd.untrusted_workload_runtime" is a runtime to run untrusted workloads on it.
+    [plugins.cri.containerd.untrusted_workload_runtime]
+      # runtime_type is the runtime type to use in containerd e.g. io.containerd.runtime.v1.linux
+      runtime_type = "io.containerd.runtime.v1.linux"
+
+      # runtime_engine is the name of the runtime engine used by containerd.
+      runtime_engine = "/usr/bin/kata-runtime"
+```
+
 You can find more information on the [Containerd config documentation](https://github.com/containerd/cri/blob/master/docs/config.md)
 
+
 #### Kata Containers as the default runtime
 
 If you want to set Kata Containers as the only runtime in the deployment, you can simply configure as follows:
@@ -197,6 +250,15 @@ If you want to set Kata Containers as the only runtime in the deployment, you ca
       runtime_type = "io.containerd.kata.v2"
 ```
 
+Alternatively, for the earlier versions of Kata Containers and containerd that do not support Runtime V2 (Shim API), you can use the following alternative configuration:
+
+```toml
+    [plugins.cri.containerd]
+    [plugins.cri.containerd.default_runtime]
+      runtime_type = "io.containerd.runtime.v1.linux"
+      runtime_engine = "/usr/bin/kata-runtime"
+```
+
 ### Configuration for `cri-tools`
 
 > **Note:** If you skipped the [Install `cri-tools`](#install-cri-tools) section, you can skip this section too.
@@ -250,12 +312,10 @@ To run a container with Kata Containers through the containerd command line, you
 
 ```bash
 $ sudo ctr image pull docker.io/library/busybox:latest
-$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
+$ sudo ctr run --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
 ```
 
 This launches a BusyBox container named `hello`, and it will be removed by `--rm` after it quits.
-The `--cni` flag enables CNI networking for the container. Without this flag, a container with just a
-loopback interface is created.
 
 ### Launch Pods with `crictl` command line
 

docs/how-to/data/confidential-agent-config.toml.in

@@ -1,45 +0,0 @@
-# Copyright (c) 2021 IBM Corp.
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-aa_kbc_params = "$AA_KBC_PARAMS"
-https_proxy = "$HTTPS_PROXY"
-[endpoints]
-allowed = [
-"AddARPNeighborsRequest",
-"AddSwapRequest",
-"CloseStdinRequest",
-"CopyFileRequest",
-"CreateContainerRequest",
-"CreateSandboxRequest",
-"DestroySandboxRequest",
-#"ExecProcessRequest",
-"GetMetricsRequest",
-"GetOOMEventRequest",
-"GuestDetailsRequest",
-"ListInterfacesRequest",
-"ListRoutesRequest",
-"MemHotplugByProbeRequest",
-"OnlineCPUMemRequest",
-"PauseContainerRequest",
-"PullImageRequest",
-"ReadStreamRequest",
-"RemoveContainerRequest",
-#"ReseedRandomDevRequest",
-"ResizeVolumeRequest",
-"ResumeContainerRequest",
-"SetGuestDateTimeRequest",
-"SignalProcessRequest",
-"StartContainerRequest",
-"StartTracingRequest",
-"StatsContainerRequest",
-"StopTracingRequest",
-"TtyWinResizeRequest",
-"UpdateContainerRequest",
-"UpdateInterfaceRequest",
-"UpdateRoutesRequest",
-"VolumeStatsRequest",
-"WaitProcessRequest",
-"WriteStreamRequest"
-]

docs/how-to/data/kata-monitor-daemonset.yml

@@ -45,9 +45,6 @@ spec:
         - name: containerdsocket
           mountPath: /run/containerd/containerd.sock
           readOnly: true
-        - name: sbs
-          mountPath: /run/vc/sbs/
-          readOnly: true
       terminationGracePeriodSeconds: 30
       volumes:
       - name: containerdtask
@@ -56,6 +53,3 @@ spec:
       - name: containerdsocket
         hostPath:
           path: /run/containerd/containerd.sock
-      - name: sbs
-        hostPath:
-          path: /run/vc/sbs/

docs/how-to/how-to-build-and-test-ccv0.md

@@ -1,479 +0,0 @@
-# How to build, run and test Kata CCv0
-
-## Introduction and Background
-
-In order to try and make building (locally) and demoing the Kata Containers `CCv0` code base as simple as possible I've
-shared a script [`ccv0.sh`](./ccv0.sh). This script was originally my attempt to automate the steps of the 
-[Developer Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md) so that I could do
-different sections of them repeatedly and reliably as I was playing around with make changes to different parts of the 
-Kata code base. I then tried to weave in some of the [`tests/.ci`](https://github.com/kata-containers/tests/tree/main/.ci) 
-scripts in order to have less duplicated code.
-As we're progress on the confidential containers journey I hope to add more features to demonstrate the functionality
-we have working.
-
-*Disclaimer: This script has mostly just been used and tested by me ([@stevenhorsman](https://github.com/stevenhorsman)),*
-*so there might be issues with it. I'm happy to try and help solve these if possible, but this shouldn't be considered a*
-*fully supported process by the Kata Containers community.*
-
-### Basic script set-up and optional environment variables
-
-In order to build, configure and demo the CCv0 functionality, these are the set-up steps I take:
-- Provision a new VM
-    - *I choose a Ubuntu 20.04 8GB VM for this as I had one available. There are some dependences on apt-get installed*
-    *packages, so these will need re-working to be compatible with other platforms.*
-- Copy the script over to your VM *(I put it in the home directory)* and ensure it has execute permission by running 
-```bash
-$ chmod u+x ccv0.sh
-```
-- Optionally set up some environment variables
-    - By default the script checks out the `CCv0` branches of the `kata-containers/kata-containers` and 
-      `kata-containers/tests` repositories, but it is designed to be used to test of personal forks and branches as well. 
-      If you want to build and run these you can export the `katacontainers_repo`, `katacontainers_branch`, `tests_repo`
-      and `tests_branch` variables e.g.
-      ```bash
-      $ export katacontainers_repo=github.com/stevenhorsman/kata-containers
-      $ export katacontainers_branch=stevenh/agent-pull-image-endpoint
-      $ export tests_repo=github.com/stevenhorsman/tests
-      $ export tests_branch=stevenh/add-ccv0-changes-to-build
-      ```
-      before running the script.
-    - By default the build and configuration are using `QEMU` as the hypervisor. In order to use `Cloud Hypervisor` instead
-      set:
-      ```
-      $ export KATA_HYPERVISOR="cloud-hypervisor"
-      ```
-      before running the build.
-
-- At this point you can provision a Kata confidential containers pod and container with either
-  [`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image),
-  or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-  and then test and use it. 
-
-### Using crictl for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with Kubernetes turned off, so its configuration doesn't interfere with `crictl` using:
-  ```bash
-  $ export KUBERNETES="no"
-  $ export KATA_HYPERVISOR="qemu"
-  $ ~/ccv0.sh -d build_and_install_all
-  ```
-    > **Note**: Much of this script has to be run as `sudo`, so you are likely to get prompted for your password.
-    - *I run this script sourced just so that the required installed components are accessible on the `PATH` to the rest*
-      *of the process without having to reload the session.*
-    - The steps that `build_and_install_all` takes is:
-        - Checkout the git repos for the `tests` and `kata-containers` repos as specified by the environment variables
-        (default to `CCv0` branches if they are not supplied)
-        - Use the `tests/.ci` scripts to install the build dependencies
-        - Build and install the Kata runtime
-        - Configure Kata to use containerd and for debug and confidential containers features to be enabled (including
-          enabling console access to the Kata guest shell, which should only be done in development)
-        - Create, build and install a rootfs for the Kata hypervisor to use. For 'CCv0' this is currently based on Ubuntu
-        20.04 and has extra packages like `umoci` added.
-        - Build the Kata guest kernel
-        - Install the hypervisor (in order to select which hypervisor will be used, the `KATA_HYPERVISOR` environment
-        variable can be used to select between `qemu` or `cloud-hypervisor`)
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull ubuntu
-  >  ```
-  >  then re-run the command.
-    - The first time this runs it may take a while, but subsequent runs will be quicker as more things are already
-      installed and they can be further cut down by not running all the above steps 
-      [see "Additional script usage" below](#additional-script-usage)
-   
-- Create a new Kata sandbox pod using `crictl` with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_pod
-  ```
-    - This creates a pod configuration file, creates the pod from this using 
-    `sudo crictl runp -r kata ~/pod-config.yaml` and runs `sudo crictl pods` to show the pod
-- Create a new Kata confidential container with:
-  ```bash
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-    - This creates a container (based on `busybox:1.33.1`) in the Kata cc sandbox and prints a list of containers.
-      This will have been created based on an image pulled in the Kata pod sandbox/guest, not on the host machine.
-
-As this point you should have a `crictl` pod and container that is using the Kata confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the `crictl` pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh crictl_delete_cc
-  ```
-### Using Kubernetes for end-to-end provisioning of a Kata confidential containers pod with an unencrypted image
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM
-  as a single node cluster: 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a Kata confidential containers pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_cc_pod
-  ```
-- Wait a few seconds for pod to start then check that the pod's status is `Running` with 
-  ```bash
-  $ kubectl get pods
-  ```
-  which should show something like:
-  ```text
-  NAME         READY   STATUS    RESTARTS   AGE
-  busybox-cc   1/1     Running   0          54s
-  ```
-
-- As this point you should have a Kubernetes pod and container running, that is using the Kata 
-confidential containers runtime.
-You can [validate that the container image was pulled on the guest](#validate-that-the-container-image-was-pulled-on-the-guest)
-or [using the Kata pod sandbox for testing with `agent-ctl` or `ctr shim`](#using-a-kata-pod-sandbox-for-testing-with-agent-ctl-or-ctr-shim)
-
-#### Clean up the Kubernetes pod sandbox and container
-- When the testing is complete you can delete the container and pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_cc_pod
-  ```
-
-### Validate that the container image was pulled on the guest
-
-There are a couple of ways we can check that the container pull image action was offloaded to the guest, by checking
-the guest's file system for the unpacked bundle and checking the host's directories to ensure it wasn't also pulled
-there.
-- To check the guest's file system:
-    - Open a shell into the Kata guest with:
-      ```bash
-      $ ~/ccv0.sh open_kata_shell
-      ```
-    - List the files in the directory that the container image bundle should have been unpacked to with:
-      ```bash
-      $ ls -ltr /run/kata-containers/confidential-containers_signed/
-      ```
-    - This should give something like
-        ```
-        total 72
-        -rw-r--r--  1 root root  2977 Jan 20 10:03 config.json
-        -rw-r--r--  1 root root   372 Jan 20 10:03 umoci.json
-        -rw-r--r--  1 root root 63584 Jan 20 10:03 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree
-        drwxr-xr-x 12 root root   240 Jan 20 10:03 rootfs
-        ```
-        which shows how the image has been pulled and then unbundled on the guest.
-    - Leave the Kata guest shell by running:
-      ```bash
-      $ exit
-      ```
-- To verify that the image wasn't pulled on the host system we can look at the shared sandbox on the host and we
-  should only see a single bundle for the pause container as the `busybox` based container image should have been
-  pulled on the guest:
-    - Find all the `rootfs` directories under in the pod's shared directory with:
-      ```bash
-      $ pod_id=$(ps -ef | grep containerd-shim-kata-v2 | egrep -o "id [^,][^,].* " | awk '{print $2}')
-      $ sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs
-      ```
-      which should only show a single `rootfs` directory if the container image was pulled on the guest, not the host
-    - Looking that `rootfs` directory with
-      ```bash
-      $ sudo ls -ltr $(sudo find /run/kata-containers/shared/sandboxes/${pod_id}/shared -name rootfs)
-      ```
-      shows something similar to
-      ```
-      total 668
-      -rwxr-xr-x 1 root root 682696 Aug 25 13:58 pause
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 proc
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 dev
-      drwxr-xr-x 2 root root      6 Jan 20 02:01 sys
-      drwxr-xr-x 2 root root     25 Jan 20 02:01 etc
-      ```
-      which is clearly the pause container indicating that the `busybox` based container image is not exposed to the host.
-
-### Using a Kata pod sandbox for testing with `agent-ctl` or `ctr shim`
-
-Once you have a kata pod sandbox created as described above, either using 
-[`crictl`](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image), or [Kubernetes](#using-kubernetes-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-, you can use this to test specific components of the Kata confidential
-containers architecture. This can be useful for development and debugging to isolate and test features
-that aren't broadly supported end-to-end. Here are some examples:
-
-- In the first terminal run the pull image on guest command against the Kata agent, via the shim (`containerd-shim-kata-v2`).
-This can be achieved using the [containerd](https://github.com/containerd/containerd) CLI tool, `ctr`, which can be used to
-interact with the shim directly. The command takes the form
-`ctr --namespace k8s.io shim --id <sandbox-id> pull-image <image> <new-container-id>` and can been run directly, or through
-the `ccv0.sh` script to automatically fill in the variables:
-  - Optionally, set up some environment variables to set the image and credentials used:
-    - By default the shim pull image test in `ccv0.sh` will use the `busybox:1.33.1` based test image
-     `quay.io/kata-containers/confidential-containers:signed` which requires no authentication. To use a different
-      image, set the `PULL_IMAGE` environment variable e.g. 
-      ```bash
-      $ export PULL_IMAGE="docker.io/library/busybox:latest"
-      ```
-      Currently the containerd shim pull image
-      code doesn't support using a container registry that requires authentication, so if this is required, see the 
-      below steps to run the pull image command against the agent directly.
-  - Run the pull image agent endpoint with:
-    ```bash
-    $ ~/ccv0.sh shim_pull_image
-    ```
-    which we print the `ctr shim` command for reference
-- Alternatively you can issue the command directly to the `kata-agent` pull image endpoint, which also supports
-  credentials in order to pull from an authenticated registry:
-    - Optionally set up some environment variables to set the image and credentials used:
-        - Set the `PULL_IMAGE` environment variable e.g. `export PULL_IMAGE="docker.io/library/busybox:latest"`
-          if a specific container image is required.
-        - If the container registry for the image requires authentication then this can be set with an environment
-          variable `SOURCE_CREDS`. For example to use Docker Hub (`docker.io`) as an authenticated user first run 
-          `export SOURCE_CREDS="<dockerhub username>:<dockerhub api key>"`
-            > **Note**: the credentials support on the agent request is a tactical solution for the short-term
-              proof of concept to allow more images to be pulled and tested. Once we have support for getting
-              keys into the Kata guest image using the attestation-agent and/or KBS I'd expect container registry
-              credentials to be looked up using that mechanism.
-    - Run the pull image agent endpoint with
-      ```bash
-      $ ~/ccv0.sh agent_pull_image
-      ```
-      and you should see output which includes `Command PullImage (1 of 1) returned (Ok(()), false)` to indicate
-      that the `PullImage` request was successful e.g.
-      ```
-      Finished release [optimized] target(s) in 0.21s
-      {"msg":"announce","level":"INFO","ts":"2021-09-15T08:40:14.189360410-07:00","subsystem":"rpc","name":"kata-agent-ctl","pid":"830920","version":"0.1.0","source":"kata-agent-ctl","config":"Config { server_address: \"vsock://1970354082:1024\", bundle_dir: \"/tmp/bundle\", timeout_nano: 0, interactive: false, ignore_errors: false }"}
-      {"msg":"client setup complete","level":"INFO","ts":"2021-09-15T08:40:14.193639057-07:00","pid":"830920","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","server-address":"vsock://1970354082:1024"}
-      {"msg":"Run command PullImage (1 of 1)","level":"INFO","ts":"2021-09-15T08:40:14.196643765-07:00","pid":"830920","source":"kata-agent-ctl","subsystem":"rpc","name":"kata-agent-ctl","version":"0.1.0"}
-      {"msg":"response received","level":"INFO","ts":"2021-09-15T08:40:43.828200633-07:00","source":"kata-agent-ctl","name":"kata-agent-ctl","subsystem":"rpc","version":"0.1.0","pid":"830920","response":""}
-      {"msg":"Command PullImage (1 of 1) returned (Ok(()), false)","level":"INFO","ts":"2021-09-15T08:40:43.828261708-07:00","subsystem":"rpc","pid":"830920","source":"kata-agent-ctl","version":"0.1.0","name":"kata-agent-ctl"}
-      ```
-      > **Note**: The first time that `~/ccv0.sh agent_pull_image` is run, the `agent-ctl` tool will be built
-      which may take a few minutes.
-- To validate that the image pull was successful, you can open a shell into the Kata guest with:
-  ```bash
-  $ ~/ccv0.sh open_kata_shell
-  ```
-- Check the `/run/kata-containers/` directory to verify that the container image bundle has been created in a directory
-  named either `01234556789` (for the container id), or the container image name, e.g.
-  ```bash
-  $ ls -ltr /run/kata-containers/confidential-containers_signed/
-  ```
-  which should show something like
-  ```
-  total 72
-  drwxr-xr-x 10 root root   200 Jan  1  1970 rootfs
-  -rw-r--r--  1 root root  2977 Jan 20 16:45 config.json
-  -rw-r--r--  1 root root   372 Jan 20 16:45 umoci.json
-  -rw-r--r--  1 root root 63584 Jan 20 16:45 sha256_be9faa75035c20288cde7d2cdeb6cd1f5f4dbcd845d3f86f7feab61c4eff9eb5.mtree
-  ```
-- Leave the Kata shell by running:
-  ```bash
-  $ exit
-  ```
-
-## Verifying signed images
-
-For this sample demo, we use local attestation to pass through the required
-configuration to do container image signature verification. Due to this, the ability to verify images is limited
-to a pre-created selection of test images in our test
-repository [`quay.io/kata-containers/confidential-containers`](https://quay.io/repository/kata-containers/confidential-containers?tab=tags).
-For pulling images not in this test repository (called an *unprotected* registry below), we fall back to the behaviour
-of not enforcing signatures. More documentation on how to customise this to match your own containers through local,
-or remote attestation will be available in future.
-
-In our test repository there are three tagged images:
-
-| Test Image | Base Image used | Signature status | GPG key status |
-| --- | --- | --- | --- |
-| `quay.io/kata-containers/confidential-containers:signed` | `busybox:1.33.1` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/signatures.tar) embedded in kata rootfs |  [public key](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/public.gpg) embedded in kata rootfs |
-| `quay.io/kata-containers/confidential-containers:unsigned` | `busybox:1.33.1` | not signed | not signed |
-| `quay.io/kata-containers/confidential-containers:other_signed` | `nginx:1.21.3` | [signature](https://github.com/kata-containers/tests/tree/CCv0/integration/confidential/fixtures/quay_verification/signatures.tar) embedded in kata rootfs | GPG key not kept |
-
-Using a standard unsigned `busybox` image that can be pulled from another, *unprotected*, `quay.io` repository we can
-test a few scenarios.
-
-In this sample, with local attestation, we pass in the the public GPG key and signature files, and the [`offline_fs_kbc`
-configuration](https://github.com/confidential-containers/attestation-agent/blob/main/src/kbc_modules/offline_fs_kbc/README.md)
-into the guest image which specifies that any container image from `quay.io/kata-containers`
-must be signed with the embedded GPG key and the agent configuration needs updating to enable this. 
-With this policy set a few tests of image verification can be done to test different scenarios by attempting
-to create containers from these images using `crictl`:
-
-- If you don't already have the Kata Containers CC code built and configured for `crictl`, then follow the 
-[instructions above](#using-crictl-for-end-to-end-provisioning-of-a-kata-confidential-containers-pod-with-an-unencrypted-image)
-up to the `~/ccv0.sh crictl_create_cc_pod` command.
-
-- In order to enable the guest image, you will need to setup the required configuration, policy and signature files
-needed by running 
-`~/ccv0.sh copy_signature_files_to_guest` and then run `~/ccv0.sh crictl_create_cc_pod` which will delete and recreate 
-your pod - adding in the new files.
- 
-- To test the fallback behaviour works using an unsigned image from an *unprotected* registry we can pull the `busybox`
-image by running:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-unprotected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes showing the running container e.g.
-  ```text
-  CONTAINER           IMAGE                               CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  98c70fefe997a       quay.io/prometheus/busybox:latest   Less than a second ago   Running             prometheus-busybox-signed   0                   70119e0539238
-  ```
-- To test that an unsigned image from our *protected* test container registry is rejected we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_unsigned-protected.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This correctly results in an error message from `crictl`:
-  `PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [Match reference failed.]" image="quay.io/kata-containers/confidential-containers:unsigned"`
-- To test that the signed image our *protected* test container registry is accepted we can run:
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - This finishes by showing a new `kata-cc-busybox-signed` running container e.g.
-  ```text
-  CONTAINER           IMAGE                                                    CREATED                  STATE               NAME                        ATTEMPT             POD ID
-  b4d85c2132ed9       quay.io/kata-containers/confidential-containers:signed   Less than a second ago   Running             kata-cc-busybox-signed      0                   70119e0539238
-  ...
-  ```
-- Finally to check the image with a valid signature, but invalid GPG key (the real trusted piece of information we really
-want to protect with the attestation agent in future) fails we can run: 
-  ```bash
-  $ export CONTAINER_CONFIG_FILE=container-config_signed-protected-other.yaml
-  $ ~/ccv0.sh crictl_create_cc_container
-  ```
-  - Again this results in an error message from `crictl`:
-  `"PullImage from image service failed" err="rpc error: code = Internal desc = Security validate failed: Validate image failed: The signatures do not satisfied! Reject reason: [signature verify failed! There is no pubkey can verify the signature!]" image="quay.io/kata-containers/confidential-containers:other_signed"`
-
-### Using Kubernetes to create a Kata confidential containers pod from the encrypted ssh demo sample image
-
-The [ssh-demo](https://github.com/confidential-containers/documentation/tree/main/demos/ssh-demo) explains how to
-demonstrate creating a Kata confidential containers pod from an encrypted image with the runtime created by the
-[confidential-containers operator](https://github.com/confidential-containers/documentation/blob/main/demos/operator-demo).
-To be fully confidential, this should be run on a Trusted Execution Environment, but it can be tested on generic
-hardware as well.
-
-If you wish to build the Kata confidential containers runtime to do this yourself, then you can using the following
-steps:
-
-- Run the full build process with the Kubernetes environment variable set to `"yes"`, so the Kubernetes cluster is
-  configured and created using the VM as a single node cluster and with `AA_KBC` set to `offline_fs_kbc`. 
-  ```bash
-  $ export KUBERNETES="yes"
-  $ export AA_KBC=offline_fs_kbc
-  $ ~/ccv0.sh build_and_install_all
-  ```
-    - The `AA_KBC=offline_fs_kbc` mode will ensure that, when creating the rootfs of the Kata guest, the 
-      [attestation-agent](https://github.com/confidential-containers/attestation-agent) will be added along with the
-      [sample offline KBC](https://github.com/confidential-containers/documentation/blob/main/demos/ssh-demo/aa-offline_fs_kbc-keys.json)
-      and an agent configuration file
-    > **Note**: Depending on how where your VMs are hosted and how IPs are shared you might get an error from docker
-    during matching `ERROR: toomanyrequests: Too Many Requests`. To get past
-    this, login into Docker Hub and pull the images used with:
-  >  ```bash
-  >  $ sudo docker login
-  >  $ sudo docker pull registry:2
-  >  $ sudo docker pull ubuntu:20.04
-  >  ```
-  >  then re-run the command.
-- Check that your Kubernetes cluster has been correctly set-up by running : 
-  ```bash
-  $ kubectl get nodes
-  ```
-  and checking that you see a single node e.g.
-  ```text
-  NAME                             STATUS   ROLES                  AGE   VERSION
-  stevenh-ccv0-k8s1.fyre.ibm.com   Ready    control-plane,master   43s   v1.22.0
-  ```
-- Create a sample Kata confidential containers ssh pod by running:
-  ```bash
-  $ ~/ccv0.sh kubernetes_create_ssh_demo_pod
-  ```
-- As this point you should have a Kubernetes pod running the Kata confidential containers runtime that has pulled
-the [sample image](https://hub.docker.com/r/katadocker/ccv0-ssh) which was encrypted by the key file that we included
-in the rootfs.
-During the pod deployment the image was pulled and then decrypted using the key file, on the Kata guest image, without
-it ever being available to the host.
-
-- To validate that the container is working you, can connect to the image via SSH by running:
-  ```bash
-  $ ~/ccv0.sh connect_to_ssh_demo_pod
-  ```
-  - During this connection the host key fingerprint is shown and should match:
-    `ED25519 key fingerprint is SHA256:wK7uOpqpYQczcgV00fGCh+X97sJL3f6G1Ku4rvlwtR0.`
-  - After you are finished connecting then run:
-    ```bash
-    $ exit
-    ```
-  
-- To delete the sample SSH demo pod run:
-  ```bash
-  $ ~/ccv0.sh kubernetes_delete_ssh_demo_pod
-  ```
-
-## Additional script usage
-
-As well as being able to use the script as above to build all of `kata-containers` from scratch it can be used to just
-re-build bits of it by running the script with different parameters. For example after the first build you will often
-not need to re-install the dependencies, the hypervisor or the Guest kernel, but just test code changes made to the
-runtime and agent. This can be done by running `~/ccv0.sh rebuild_and_install_kata`. (*Note this does a hard checkout*
-*from git, so if your changes are only made locally it is better to do the individual steps e.g.* 
-`~/ccv0.sh build_kata_runtime && ~/ccv0.sh build_and_add_agent_to_rootfs && ~/ccv0.sh build_and_install_rootfs`).
-There are commands for a lot of steps in building, setting up and testing and the full list can be seen by running
-`~/ccv0.sh help`:
-```
-$ ~/ccv0.sh help
-Overview:
-    Build and test kata containers from source
-    Optionally set kata-containers and tests repo and branch as exported variables before running
-    e.g. export katacontainers_repo=github.com/stevenhorsman/kata-containers && export katacontainers_branch=kata-ci-from-fork && export tests_repo=github.com/stevenhorsman/tests && export tests_branch=kata-ci-from-fork && ~/ccv0.sh build_and_install_all
-Usage:
-    ccv0.sh [options] <command>
-Commands:
-- help:                         Display this help
-- all:                          Build and install everything, test kata with containerd and capture the logs
-- build_and_install_all:        Build and install everything
-- initialize:                   Install dependencies and check out kata-containers source
-- rebuild_and_install_kata:     Rebuild the kata runtime and agent and build and install the image
-- build_kata_runtime:           Build and install the kata runtime
-- configure:                    Configure Kata to use rootfs and enable debug
-- create_rootfs:                Create a local rootfs
-- build_and_add_agent_to_rootfs:Builds the kata-agent and adds it to the rootfs
-- build_and_install_rootfs:     Builds and installs the rootfs image
-- install_guest_kernel:         Setup, build and install the guest kernel
-- build_cloud_hypervisor        Checkout, patch, build and install Cloud Hypervisor
-- build_qemu:                   Checkout, patch, build and install QEMU
-- init_kubernetes:              initialize a Kubernetes cluster on this system
-- crictl_create_cc_pod          Use crictl to create a new kata cc pod
-- crictl_create_cc_container    Use crictl to create a new busybox container in the kata cc pod
-- crictl_delete_cc              Use crictl to delete the kata cc pod sandbox and container in it
-- kubernetes_create_cc_pod:     Create a Kata CC runtime busybox-based pod in Kubernetes
-- kubernetes_delete_cc_pod:     Delete the Kata CC runtime busybox-based pod in Kubernetes
-- open_kata_shell:              Open a shell into the kata runtime
-- agent_pull_image:             Run PullImage command against the agent with agent-ctl
-- shim_pull_image:              Run PullImage command against the shim with ctr
-- agent_create_container:       Run CreateContainer command against the agent with agent-ctl
-- test:                         Test using kata with containerd
-- test_capture_logs:            Test using kata with containerd and capture the logs in the user's home directory
-
-Options:
-    -d: Enable debug
-    -h: Display this help
-```

docs/how-to/how-to-generate-a-kata-containers-payload-for-the-confidential-containers-operator.md

@@ -1,44 +0,0 @@
-# Generating a Kata Containers payload for the Confidential Containers Operator
-
-[Confidential Containers
-Operator](https://github.com/confidential-containers/operator) consumes a Kata
-Containers payload, generated from the `CCv0` branch, and here one can find all
-the necessary info on how to build such a payload.
-
-## Requirements
-
-* `make` installed in the machine
-* Docker installed in the machine
-* `sudo` access to the machine
-
-## Process
-
-* Clone [Kata Containers](https://github.com/kata-containers/kata-containers)
-  ```sh
-  git clone --branch CCv0 https://github.com/kata-containers/kata-containers
-  ```
-  * In case you've already cloned the repo, make sure to switch to the `CCv0` branch
-    ```sh
-    git checkout CCv0
-    ```
-  * Ensure your tree is clean and in sync with upstream `CCv0`
-    ```sh
-    git clean -xfd
-    git reset --hard <upstream>/CCv0
-    ```
-* Make sure you're authenticated to `quay.io`
-    ```sh
-    sudo docker login quay.io
-    ```
-* From the top repo directory, run:
-  ```sh
-  sudo make cc-payload
-  ```
-* Make sure the image was upload to the [Confidential Containers
-  runtime-payload
-registry](https://quay.io/repository/confidential-containers/runtime-payload?tab=tags)
-
-## Notes
-
-Make sure to run it on a machine that's not the one you're hacking on, prepare a
-cup of tea, and get back to it an hour later (at least).

docs/how-to/how-to-import-kata-logs-with-fluentd.md

@@ -68,7 +68,7 @@ the Kata logs import to the EFK stack.
 > stack they are able to utilise in order to modify and test as necessary.
 
 Minikube by default
-[configures](https://github.com/kubernetes/minikube/blob/master/deploy/iso/minikube-iso/board/minikube/x86_64/rootfs-overlay/etc/systemd/journald.conf)
+[configures](https://github.com/kubernetes/minikube/blob/master/deploy/iso/minikube-iso/board/coreos/minikube/rootfs-overlay/etc/systemd/journald.conf)
  the `systemd-journald` with the
 [`Storage=volatile`](https://www.freedesktop.org/software/systemd/man/journald.conf.html) option,
 which results in the journal being stored in `/run/log/journal`. Unfortunately, the Minikube EFK
@@ -163,7 +163,7 @@ sub-filter on, for instance, the `SYSLOG_IDENTIFIER` to differentiate the Kata c
 on the `PRIORITY` to filter out critical issues etc.
 
 Kata generates a significant amount of Kata specific information, which can be seen as
-[`logfmt`](../../src/tools/log-parser/README.md#logfile-requirements).
+[`logfmt`](https://github.com/kata-containers/tests/tree/main/cmd/log-parser#logfile-requirements).
 data contained in the `MESSAGE` field. Imported as-is, there is no easy way to filter on that data
 in Kibana:
 

docs/how-to/how-to-run-docker-with-kata.md

@@ -48,9 +48,9 @@ Running Docker containers Kata Containers requires care because `VOLUME`s specif
 kataShared on / type virtiofs (rw,relatime,dax)
 ```
 
-`kataShared` mount types are powered by [`virtio-fs`](https://virtio-fs.gitlab.io/), a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
+`kataShared` mount types are powered by [`virtio-fs`][virtio-fs], a marked improvement over `virtio-9p`, thanks to [PR #1016](https://github.com/kata-containers/runtime/pull/1016). While `virtio-fs` is normally an excellent choice, in the case of DinD workloads `virtio-fs` causes an issue -- [it *cannot* be used as a "upper layer" of `overlayfs` without a custom patch](http://lists.katacontainers.io/pipermail/kata-dev/2020-January/001216.html).
 
-As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` will fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
+As `/var/lib/docker` is a `VOLUME` specified by DinD (i.e. the `docker` images tagged `*-dind`/`*-dind-rootless`), `docker` fill fail to start (or even worse, silently pick a worse storage driver like `vfs`) when started in a Kata Container. Special measures must be taken when running DinD-powered workloads in Kata Containers.
 
 ## Workarounds/Solutions
 
@@ -58,7 +58,7 @@ Thanks to various community contributions (see [issue references below](#referen
 
 ### Use a memory backed volume
 
-For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume](https://kubernetes.io/docs/concepts/storage/volumes/#emptydir), which allows for memdisk-backed storage -- the the `medium: Memory`. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
+For small workloads (small container images, without much generated filesystem load), a memory-backed volume is sufficient. Kubernetes supports a variant of  [the `EmptyDir` volume][k8s-emptydir], which allows for memdisk-backed storage -- the [the `medium: Memory` ][k8s-memory-volume-type]. An example of a `Pod` using such a setup [was contributed](https://github.com/kata-containers/runtime/issues/1429#issuecomment-477385283), and is reproduced below:
 
 ```yaml
 apiVersion: v1

docs/how-to/how-to-run-kata-containers-with-SNP-VMs.md

@@ -1,159 +0,0 @@
-# Kata Containers with AMD SEV-SNP VMs
-
-## Disclaimer
-
-This guide is designed for developers and is - same as the Developer Guide - not intended for production systems or end users. It is advisable to only follow this guide on non-critical development systems.
-
-## Prerequisites
-
-To run  Kata Containers in SNP-VMs, the following software stack is used.
-
-![Kubernetes integration with shimv2](./images/SNP-stack.svg)
-
-The host BIOS and kernel must be capable of supporting AMD SEV-SNP and configured accordingly. For Kata Containers, the host kernel with branch [`sev-snp-iommu-avic_5.19-rc6_v3`](https://github.com/AMDESE/linux/tree/sev-snp-iommu-avic_5.19-rc6_v3) and commit [`3a88547`](https://github.com/AMDESE/linux/commit/3a885471cf89156ea555341f3b737ad2a8d9d3d0) is known to work in conjunction with SEV Firmware version 1.51.3 (0xh\_1.33.03) available on AMD's [SEV developer website](https://developer.amd.com/sev/). See [AMD's guide](https://github.com/AMDESE/AMDSEV/tree/sev-snp-devel) to configure the host accordingly. Verify that you are able to run SEV-SNP encrypted VMs first. The guest components required for Kata Containers are built as described below.
-
-**Tip**: It is easiest to first have Kata Containers running on your system and then modify it to run containers in SNP-VMs. Follow the [Developer guide](../Developer-Guide.md#warning) and then follow the below steps. Nonetheless, you can just follow this guide from the start.
-
-## How to build
-
-Follow all of the below steps to install Kata Containers with SNP-support from scratch. These steps mostly follow the developer guide with modifications to support SNP
-
-__Steps from the Developer Guide:__
-- Get all the [required components](../Developer-Guide.md#requirements-to-build-individual-components) for building the kata-runtime
-- [Build the and install kata-runtime](../Developer-Guide.md#build-and-install-the-kata-containers-runtime)
-- [Build a custom agent](../Developer-Guide.md#build-a-custom-kata-agent---optional)
-- [Create an initrd image](../Developer-Guide.md#create-an-initrd-image---optional) by first building a rootfs, then building the initrd based on the rootfs, use a custom agent and install. `ubuntu` works as the distribution of choice.
-- Get the [required components](../../tools/packaging/kernel/README.md#requirements) to build a custom kernel
-
-__SNP-specific steps:__
-- Build the SNP-specific kernel as shown below (see this [guide](../../tools/packaging/kernel/README.md#build-kata-containers-kernel) for more information)
-```bash
-$ pushd kata-containers/tools/packaging/kernel/
-$ ./build-kernel.sh -a x86_64 -x snp setup
-$ ./build-kernel.sh -a x86_64 -x snp build
-$ sudo -E PATH="${PATH}" ./build-kernel.sh -x snp install
-$ popd
-```
-- Build a current OVMF capable of SEV-SNP:
-```bash
-$ pushd kata-containers/tools/packaging/static-build/ovmf
-$ ./build.sh
-$ tar -xvf edk2-x86_64.tar.gz
-$ popd
-```
-- Build a custom QEMU
-```bash
-$ source kata-containers/tools/packaging/scripts/lib.sh
-$ qemu_url="$(get_from_kata_deps "assets.hypervisor.qemu.snp.url")"
-$ qemu_branch="$(get_from_kata_deps "assets.hypervisor.qemu.snp.branch")"
-$ qemu_commit="$(get_from_kata_deps "assets.hypervisor.qemu.snp.commit")"
-$ git clone -b "${qemu_branch}" "${qemu_url}"
-$ pushd qemu
-$ git checkout "${qemu_commit}"
-$ ./configure --enable-virtfs --target-list=x86_64-softmmu --enable-debug
-$ make -j "$(nproc)"
-$ popd
-```
-
-### Kata Containers Configuration for SNP
-
-The configuration file located at `/etc/kata-containers/configuration.toml` must be adapted as follows to support SNP-VMs:
-- Use the SNP-specific kernel for the guest VM (change path)
-```toml
-kernel = "/usr/share/kata-containers/vmlinuz-snp.container"
-```
-- Enable the use of an initrd (uncomment)
-```toml
-initrd = "/usr/share/kata-containers/kata-containers-initrd.img"
-```
-- Disable the use of a rootfs (comment out)
-```toml
-# image = "/usr/share/kata-containers/kata-containers.img"
-```
-- Use the custom QEMU capable of SNP (change path)
-```toml
-path = "/path/to/qemu/build/qemu-system-x86_64"
-```
-- Use `virtio-9p` device since `virtio-fs` is unsupported due to bugs / shortcomings in QEMU version [`snp-v3`](https://github.com/AMDESE/qemu/tree/snp-v3) for SEV and SEV-SNP (change value)
-```toml
-shared_fs = "virtio-9p"
-```
-- Disable `virtiofsd` since it is no longer required (comment out)
-```toml
-# virtio_fs_daemon = "/usr/libexec/virtiofsd"
-```
-- Disable NVDIMM (uncomment)
-```toml
-disable_image_nvdimm = true
-```
-- Disable shared memory (uncomment)
-```toml
-file_mem_backend = ""
-```
-- Enable confidential guests (uncomment)
-```toml
-confidential_guest = true
-```
-- Enable SNP-VMs (uncomment)
-```toml
-sev_snp_guest = true
-```
-  - Configure an OVMF (add path)
-```toml
-firmware = "/path/to/kata-containers/tools/packaging/static-build/ovmf/opt/kata/share/ovmf/OVMF.fd"
-```
-
-## Test Kata Containers with Containerd
-
-With Kata Containers configured to support SNP-VMs, we use containerd to test and deploy containers in these VMs.
-
-### Install Containerd
-If not already present, follow [this guide](./containerd-kata.md#install) to install containerd and its related components including `CNI` and the `cri-tools` (skip Kata Containers since we already installed it)
-
-### Containerd Configuration
-
-Follow [this guide](./containerd-kata.md#configuration) to configure containerd to use Kata Containers
-
-## Run Kata Containers in SNP-VMs
-
-Run the below commands to start a container. See [this guide](./containerd-kata.md#run) for more information
-```bash
-$ sudo ctr image pull docker.io/library/busybox:latest
-$ sudo ctr run --cni --runtime io.containerd.run.kata.v2 -t --rm docker.io/library/busybox:latest hello sh
-```
-
-### Check for active SNP:
-
-Inside the running container, run the following commands to check if SNP is active. It should look something like this:
-```
-/ # dmesg | grep -i sev
-[    0.299242] Memory Encryption Features active: AMD SEV SEV-ES SEV-SNP
-[    0.472286] SEV: Using SNP CPUID table, 31 entries present.
-[    0.514574] SEV: SNP guest platform device initialized.
-[    0.885425] sev-guest sev-guest: Initialized SEV guest driver (using vmpck_id 0)
-```
-
-### Obtain an SNP Attestation Report
-
-To obtain an attestation report inside the container, the `/dev/sev-guest` must first be configured. As of now, the VM does not perform this step, however it can be performed inside the container, either in the terminal or in code.
-
-Example for shell:
-```
-/ # SNP_MAJOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $1}')
-/ # SNP_MINOR=$(cat /sys/devices/virtual/misc/sev-guest/dev | awk -F: '{print $2}')
-/ # mknod -m 600 /dev/sev-guest c "${SNP_MAJOR}" "${SNP_MINOR}"
-```
-
-## Known Issues
-
-- Support for cgroups v2 is still [work in progress](https://github.com/kata-containers/kata-containers/issues/927). If issues occur due to cgroups v2 becoming the default in newer systems, one possible solution is to downgrade cgroups to v1:
-```bash
-sudo sed -i 's/^\(GRUB_CMDLINE_LINUX=".*\)"/\1 systemd.unified_cgroup_hierarchy=0"/' /etc/default/grub
-sudo update-grub
-sudo reboot
-```
-- If both SEV and SEV-SNP are supported by the host, Kata Containers uses SEV-SNP by default. You can verify what features are enabled by checking `/sys/module/kvm_amd/parameters/sev` and `sev_snp`. This means that Kata Containers can not run both SEV-SNP-VMs and SEV-VMs at the same time. If SEV is to be used by Kata Containers instead, reload the `kvm_amd` kernel module without SNP-support, this will disable SNP-support for the entire platform.
-```bash
-sudo rmmod kvm_amd && sudo modprobe kvm_amd sev_snp=0
-```
-

docs/how-to/how-to-set-prometheus-in-k8s.md

@@ -19,7 +19,7 @@ Also you should ensure that `kubectl` working correctly.
 > **Note**: More information about Kubernetes integrations:
 >   - [Run Kata Containers with Kubernetes](run-kata-with-k8s.md)
 >   - [How to use Kata Containers and Containerd](containerd-kata.md)
->   - [How to use Kata Containers and containerd with Kubernetes](how-to-use-k8s-with-containerd-and-kata.md)
+>   - [How to use Kata Containers and CRI (containerd plugin) with Kubernetes](how-to-use-k8s-with-cri-containerd-and-kata.md)
 
 ## Configure Prometheus
 

docs/how-to/how-to-set-sandbox-config-kata.md

@@ -91,7 +91,6 @@ There are several kinds of Kata configurations and they are listed below.
 | `io.katacontainers.config.hypervisor.virtio_fs_daemon` | string | virtio-fs `vhost-user` daemon path |
 | `io.katacontainers.config.hypervisor.virtio_fs_extra_args` | string | extra options passed to `virtiofs` daemon |
 | `io.katacontainers.config.hypervisor.enable_guest_swap` | `boolean` | enable swap in the guest |
-| `io.katacontainers.config.hypervisor.use_legacy_serial` | `boolean` | uses legacy serial device for guest's console (QEMU) |
 
 ## Container Options
 | Key | Value Type | Comments |
@@ -173,7 +172,7 @@ kind: Pod
 metadata:
   name: pod2
   annotations:
-    io.katacontainers.config.runtime.disable_guest_seccomp: "false"
+    io.katacontainers.config.runtime.disable_guest_seccomp: false
 spec:
   runtimeClassName: kata
   containers:

docs/how-to/how-to-setup-swap-devices-in-guest-kernel.md

@@ -27,8 +27,6 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
-  uid: $(uuidgen)
-  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/how-to-use-k8s-with-cri-containerd-and-kata.md

@@ -1,15 +1,15 @@
-# How to use Kata Containers and containerd with Kubernetes
+# How to use Kata Containers and CRI (containerd plugin) with Kubernetes
 
 This document describes how to set up a single-machine Kubernetes (k8s) cluster.
 
 The Kubernetes cluster will use the
-[containerd](https://github.com/containerd/containerd/) and
-[Kata Containers](https://katacontainers.io) to launch workloads.
+[CRI containerd](https://github.com/containerd/containerd/) and
+[Kata Containers](https://katacontainers.io) to launch untrusted workloads.
 
 ## Requirements
 
 - Kubernetes, Kubelet, `kubeadm`
-- containerd
+- containerd with `cri` plug-in
 - Kata Containers
 
 > **Note:** For information about the supported versions of these components,
@@ -149,7 +149,7 @@ $ sudo -E kubectl taint nodes --all node-role.kubernetes.io/master-
 
 ## Create runtime class for Kata Containers
 
-By default, all pods are created with the default runtime configured in containerd.
+By default, all pods are created with the default runtime configured in CRI containerd plugin.
 From Kubernetes v1.12, users can use [`RuntimeClass`](https://kubernetes.io/docs/concepts/containers/runtime-class/#runtime-class) to specify a different runtime for Pods.
 
 ```bash
@@ -166,7 +166,7 @@ $ sudo -E kubectl apply -f runtime.yaml
 
 ## Run pod in Kata Containers
 
-If a pod has the `runtimeClassName` set to `kata`, the CRI runs the pod with the
+If a pod has the `runtimeClassName` set to `kata`, the CRI plugin runs the pod with the
 [Kata Containers runtime](../../src/runtime/README.md).
 
 - Create an pod configuration that using Kata Containers runtime

docs/how-to/how-to-use-kata-containers-with-acrn.md

@@ -101,7 +101,7 @@ Start an ACRN based Kata Container,
 $ sudo docker run -ti --runtime=kata-runtime busybox sh
 ```
 
-You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
+You will see ACRN(`acrn-dm`) is now running on your system, as well as a `kata-shim`, `kata-proxy`. You should obtain an interactive shell prompt. Verify that all the Kata processes terminate once you exit the container.
 
 ```bash
 $ ps -ef | grep -E "kata|acrn"

docs/how-to/how-to-use-kata-containers-with-firecracker.md

@@ -1,254 +0,0 @@
-# Configure Kata Containers to use Firecracker
-
-This document provides an overview on how to run Kata Containers with the AWS Firecracker hypervisor.
-
-## Introduction
-
-AWS Firecracker is an open source virtualization technology that is purpose-built for creating and managing secure, multi-tenant container and function-based services that provide serverless operational models. AWS Firecracker runs workloads in lightweight virtual machines, called `microVMs`, which combine the security and isolation properties provided by hardware virtualization technology with the speed and flexibility of Containers.
-
-Please refer to AWS Firecracker [documentation](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md) for more details.
-
-## Pre-requisites
-
-This document requires the presence of Kata Containers on your system. Install using the instructions available through the following links:
-
-- Kata Containers [automated installation](../install/README.md)
-
-- Kata Containers manual installation: Automated installation does not seem to be supported for Clear Linux, so please use [manual installation](../Developer-Guide.md) steps.
-> **Note:** Create rootfs image and not initrd image.
-
-## Install AWS Firecracker
-
-Kata Containers only support AWS Firecracker v0.23.4 ([yet](https://github.com/kata-containers/kata-containers/pull/1519)).
-To install Firecracker we need to get the `firecracker` and `jailer` binaries:
-
-```bash
-$ release_url="https://github.com/firecracker-microvm/firecracker/releases"
-$ version="v0.23.1"
-$ arch=`uname -m`
-$ curl ${release_url}/download/${version}/firecracker-${version}-${arch} -o firecracker
-$ curl ${release_url}/download/${version}/jailer-${version}-${arch} -o jailer
-$ chmod +x jailer firecracker
-```
-
-To make the binaries available from the default system `PATH` it is recommended to move them to `/usr/local/bin` or add a symbolic link:
-
-```bash
-$ sudo ln -s $(pwd)/firecracker /usr/local/bin
-$ sudo ln -s $(pwd)/jailer /usr/local/bin
-```
-
-More details can be found in [AWS Firecracker docs](https://github.com/firecracker-microvm/firecracker/blob/main/docs/getting-started.md)
-
-In order to run Kata with AWS Firecracker a block device as the backing store for a VM is required. To interact with `containerd` and Kata we use the `devmapper` `snapshotter`.
-
-## Configure `devmapper`
-
-To check support for your `containerd` installation, you can run:
-
-```
-$ ctr plugins ls |grep devmapper
-```
-
-if the output of the above command is:
-
-```
-io.containerd.snapshotter.v1    devmapper                linux/amd64    ok
-```
-then you can skip this section and move on to `Configure Kata Containers with AWS Firecracker`
-
-If the output of the above command is:
-
-```
-io.containerd.snapshotter.v1    devmapper                linux/amd64    error
-```
-
-then we need to setup `devmapper` `snapshotter`. Based on a [very useful
-guide](https://docs.docker.com/storage/storagedriver/device-mapper-driver/)
-from docker, we can set it up using the following scripts:
-
-> **Note:** The following scripts assume a 100G sparse file for storing container images, a 10G sparse file for the thin-provisioning pool and 10G base image files for any sandboxed container created. This means that we will need at least 10GB free space.
-
-```
-#!/bin/bash
-set -ex
-
-DATA_DIR=/var/lib/containerd/devmapper
-POOL_NAME=devpool
-
-mkdir -p ${DATA_DIR}
-
-# Create data file
-sudo touch "${DATA_DIR}/data"
-sudo truncate -s 100G "${DATA_DIR}/data"
-
-# Create metadata file
-sudo touch "${DATA_DIR}/meta"
-sudo truncate -s 10G "${DATA_DIR}/meta"
-
-# Allocate loop devices
-DATA_DEV=$(sudo losetup --find --show "${DATA_DIR}/data")
-META_DEV=$(sudo losetup --find --show "${DATA_DIR}/meta")
-
-# Define thin-pool parameters.
-# See https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt for details.
-SECTOR_SIZE=512
-DATA_SIZE="$(sudo blockdev --getsize64 -q ${DATA_DEV})"
-LENGTH_IN_SECTORS=$(bc <<< "${DATA_SIZE}/${SECTOR_SIZE}")
-DATA_BLOCK_SIZE=128
-LOW_WATER_MARK=32768
-
-# Create a thin-pool device
-sudo dmsetup create "${POOL_NAME}" \
-    --table "0 ${LENGTH_IN_SECTORS} thin-pool ${META_DEV} ${DATA_DEV} ${DATA_BLOCK_SIZE} ${LOW_WATER_MARK}"
-
-cat << EOF
-#
-# Add this to your config.toml configuration file and restart `containerd` daemon
-#
-[plugins]
-  [plugins.devmapper]
-    pool_name = "${POOL_NAME}"
-    root_path = "${DATA_DIR}"
-    base_image_size = "10GB"
-    discard_blocks = true
-EOF
-```
-
-Make it executable and run it:
-
-```bash
-$ sudo chmod +x ~/scripts/devmapper/create.sh
-$ cd ~/scripts/devmapper/
-$ sudo ./create.sh
-```
-
-Now, we can add the `devmapper` configuration provided from the script to `/etc/containerd/config.toml`.
-> **Note:** If you are using the default `containerd` configuration (`containerd config default >> /etc/containerd/config.toml`), you may need to edit the existing `[plugins."io.containerd.snapshotter.v1.devmapper"]`configuration.
-Save and restart `containerd`:
-
-
-```bash
-$ sudo systemctl restart containerd
-```
-
-We can use `dmsetup` to verify that the thin-pool was created successfully.
-
-```bash
-$ sudo dmsetup ls
-```
-
- We should also check that `devmapper` is registered and running:
-
-```bash
-$ sudo ctr plugins ls | grep devmapper
-```
-
-This script needs to be run only once, while setting up the `devmapper` `snapshotter` for `containerd`. Afterwards, make sure that on each reboot, the thin-pool is initialized from the same data directory. Otherwise, all the fetched containers (or the ones that you have created) will be re-initialized. A simple script that re-creates the thin-pool from the same data directory is shown below:
-
-```
-#!/bin/bash
-set -ex
-
-DATA_DIR=/var/lib/containerd/devmapper
-POOL_NAME=devpool
-
-# Allocate loop devices
-DATA_DEV=$(sudo losetup --find --show "${DATA_DIR}/data")
-META_DEV=$(sudo losetup --find --show "${DATA_DIR}/meta")
-
-# Define thin-pool parameters.
-# See https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt for details.
-SECTOR_SIZE=512
-DATA_SIZE="$(sudo blockdev --getsize64 -q ${DATA_DEV})"
-LENGTH_IN_SECTORS=$(bc <<< "${DATA_SIZE}/${SECTOR_SIZE}")
-DATA_BLOCK_SIZE=128
-LOW_WATER_MARK=32768
-
-# Create a thin-pool device
-sudo dmsetup create "${POOL_NAME}" \
-    --table "0 ${LENGTH_IN_SECTORS} thin-pool ${META_DEV} ${DATA_DEV} ${DATA_BLOCK_SIZE} ${LOW_WATER_MARK}"
-```
-
-We can create a systemd service to run the above script on each reboot:
-
-```bash
-$ sudo nano /lib/systemd/system/devmapper_reload.service
-```
-
-The service file:
-
-```
-[Unit]
-Description=Devmapper reload script
-
-[Service]
-ExecStart=/path/to/script/reload.sh
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Enable the newly created service:
-
-```bash
-$ sudo systemctl daemon-reload
-$ sudo systemctl enable devmapper_reload.service
-$ sudo systemctl start devmapper_reload.service
-```
-
-## Configure Kata Containers with AWS Firecracker
-
-To configure Kata Containers with AWS Firecracker, copy the generated `configuration-fc.toml` file when building the `kata-runtime` to either `/etc/kata-containers/configuration-fc.toml` or `/usr/share/defaults/kata-containers/configuration-fc.toml`.
-
-The following command shows full paths to the `configuration.toml` files that the runtime loads. It will use the first path that exists. (Please make sure the kernel and image paths are set correctly in the `configuration.toml` file)
-
-```bash
-$ sudo kata-runtime --show-default-config-paths
-```
-
-## Configure `containerd`
-Next, we need to configure containerd. Add a file in your path (e.g. `/usr/local/bin/containerd-shim-kata-fc-v2`) with the following contents:
-
-```
-#!/bin/bash
-KATA_CONF_FILE=/etc/containers/configuration-fc.toml /usr/local/bin/containerd-shim-kata-v2 $@
-```
-> **Note:** You may need to edit the paths of the configuration file and the `containerd-shim-kata-v2` to correspond to your setup.
-
-Make it executable:
-
-```bash
-$ sudo chmod +x /usr/local/bin/containerd-shim-kata-fc-v2
-```
-
-Add the relevant section in `containerd`’s `config.toml` file (`/etc/containerd/config.toml`):
-
-```
-[plugins.cri.containerd.runtimes]
-  [plugins.cri.containerd.runtimes.kata-fc]
-    runtime_type = "io.containerd.kata-fc.v2"
-```
-
-> **Note:** If you are using the default `containerd` configuration (`containerd config default >> /etc/containerd/config.toml`),
-> the configuration should change to :
-```
-[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
-  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.kata-fc]
-    runtime_type = "io.containerd.kata-fc.v2"
-```
-
-Restart `containerd`:
-
-```bash
-$ sudo systemctl restart containerd
-```
-
-## Verify the installation
-
-We are now ready to launch a container using Kata with Firecracker to verify that everything worked:
-
-```bash
-$ sudo ctr images pull --snapshotter devmapper docker.io/library/ubuntu:latest
-$ sudo ctr run --snapshotter devmapper --runtime io.containerd.run.kata-fc.v2 -t --rm docker.io/library/ubuntu
-```

docs/how-to/how-to-use-virtio-fs-nydus-with-kata.md

@@ -32,7 +32,6 @@ The `nydus-sandbox.yaml` looks like below:
 metadata:
   attempt: 1
   name: nydus-sandbox
-  uid: nydus-uid
   namespace: default
 log_directory: /tmp
 linux:

docs/how-to/how-to-use-virtio-mem-with-kata.md

@@ -42,8 +42,6 @@ $ image="quay.io/prometheus/busybox:latest"
 $ cat << EOF > "${pod_yaml}"
 metadata:
   name: busybox-sandbox1
-  uid: $(uuidgen)
-  namespace: default
 EOF
 $ cat << EOF > "${container_yaml}"
 metadata:

docs/how-to/privileged.md

@@ -31,7 +31,7 @@ See below example config:
   [plugins.cri]
     [plugins.cri.containerd]
        [plugins.cri.containerd.runtimes.runc]
-         runtime_type = "io.containerd.runc.v2"
+         runtime_type = "io.containerd.runc.v1"
          privileged_without_host_devices = false
        [plugins.cri.containerd.runtimes.kata]
          runtime_type = "io.containerd.kata.v2"
@@ -40,7 +40,7 @@ See below example config:
            ConfigPath = "/opt/kata/share/defaults/kata-containers/configuration.toml"
 ```
 
- - [How to use Kata Containers and containerd with Kubernetes](how-to-use-k8s-with-containerd-and-kata.md)
+ - [Kata Containers with Containerd and CRI documentation](how-to-use-k8s-with-cri-containerd-and-kata.md)
  - [Containerd CRI config documentation](https://github.com/containerd/containerd/blob/main/docs/cri/config.md)
 
 #### CRI-O

docs/how-to/run-kata-with-k8s.md

@@ -15,7 +15,7 @@ After choosing one CRI implementation, you must make the appropriate configurati
 to ensure it integrates with Kata Containers.
 
 Kata Containers 1.5 introduced the `shimv2` for containerd 1.2.0, reducing the components
-required to spawn pods and containers, and this is the preferred way to run Kata Containers with Kubernetes ([as documented here](../how-to/how-to-use-k8s-with-containerd-and-kata.md#configure-containerd-to-use-kata-containers)).
+required to spawn pods and containers, and this is the preferred way to run Kata Containers with Kubernetes ([as documented here](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#configure-containerd-to-use-kata-containers)).
 
 An equivalent shim implementation for CRI-O is planned.
 
@@ -57,7 +57,7 @@ content shown below:
 
 To customize containerd to select Kata Containers runtime, follow our
 "Configure containerd to use Kata Containers" internal documentation
-[here](../how-to/how-to-use-k8s-with-containerd-and-kata.md#configure-containerd-to-use-kata-containers).
+[here](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#configure-containerd-to-use-kata-containers).
 
 ## Install Kubernetes
 
@@ -85,7 +85,7 @@ Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --runtime-request-tim
 Environment="KUBELET_EXTRA_ARGS=--container-runtime=remote --runtime-request-timeout=15m --container-runtime-endpoint=unix:///run/containerd/containerd.sock"
 ```
 For more information about containerd see the "Configure Kubelet to use containerd"
-documentation [here](../how-to/how-to-use-k8s-with-containerd-and-kata.md#configure-kubelet-to-use-containerd).
+documentation [here](../how-to/how-to-use-k8s-with-cri-containerd-and-kata.md#configure-kubelet-to-use-containerd).
 
 ## Run a Kubernetes pod with Kata Containers
 
@@ -99,18 +99,7 @@ $ sudo systemctl restart kubelet
 $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16
 
 # If using containerd
-$ cat <<EOF | tee kubeadm-config.yaml
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: InitConfiguration
-nodeRegistration:
-  criSocket: "/run/containerd/containerd.sock"
----
-kind: KubeletConfiguration
-apiVersion: kubelet.config.k8s.io/v1beta1
-cgroupDriver: cgroupfs
-podCIDR: "10.244.0.0/16"
-EOF
-$ sudo kubeadm init --ignore-preflight-errors=all --config kubeadm-config.yaml
+$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16
 
 $ export KUBECONFIG=/etc/kubernetes/admin.conf
 ```

docs/hypervisors.md

@@ -33,7 +33,6 @@ are available, their default values and how each setting can be used.
 [Cloud Hypervisor] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-clh.toml` |
 [Firecracker] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-fc.toml` |
 [QEMU] | C | all | Type 2 ([KVM]) | `configuration-qemu.toml` |
-[`Dragonball`] | rust | `aarch64`, `x86_64` | Type 2 ([KVM]) | `configuration-dragonball.toml` |
 
 ## Determine currently configured hypervisor
 
@@ -53,7 +52,6 @@ the hypervisors:
 [Cloud Hypervisor] | Low latency, small memory footprint, small attack surface | Minimal | | excellent | excellent | High performance modern cloud workloads | |
 [Firecracker] | Very slimline | Extremely minimal | Doesn't support all device types | excellent | excellent | Serverless / FaaS | |
 [QEMU] | Lots of features | Lots | | good | good | Good option for most users | | All users |
-[`Dragonball`] | Built-in VMM,  low CPU and memory overhead| Minimal | | excellent | excellent | Optimized for most container workloads | `out-of-the-box` Kata Containers experience |
 
 For further details, see the [Virtualization in Kata Containers](design/virtualization.md) document and the official documentation for each hypervisor.
 
@@ -62,4 +60,3 @@ For further details, see the [Virtualization in Kata Containers](design/virtuali
 [Firecracker]: https://github.com/firecracker-microvm/firecracker
 [KVM]: https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine
 [QEMU]: http://www.qemu-project.org
-[`Dragonball`]: https://github.com/openanolis/dragonball-sandbox

docs/install/README.md

@@ -79,6 +79,3 @@ versions. This is not recommended for normal users.
 * [upgrading document](../Upgrading.md)
 * [developer guide](../Developer-Guide.md)
 * [runtime documentation](../../src/runtime/README.md)
-
-## Kata Containers 3.0 rust runtime installation
-* [installation guide](../install/kata-containers-3.0-rust-runtime-installation-guide.md)

docs/install/container-manager/containerd/containerd-install.md

@@ -19,6 +19,12 @@
 > - If you decide to proceed and install a Kata Containers release, you can
 >   still check for the latest version of Kata Containers by running
 >   `kata-runtime check --only-list-releases`.
+>
+> - These instructions will not work for Fedora 31 and higher since those
+>   distribution versions only support cgroups version 2 by default. However,
+>   Kata Containers currently requires cgroups version 1 (on the host side). See
+>   https://github.com/kata-containers/kata-containers/issues/927 for further
+>   details.
 
 ## Install Kata Containers
 
@@ -75,7 +81,7 @@
   - Download the standard `systemd(1)` service file and install to
     `/etc/systemd/system/`:
 
-    - https://raw.githubusercontent.com/containerd/containerd/main/containerd.service
+    - https://raw.githubusercontent.com/containerd/containerd/master/containerd.service
 
     > **Notes:**
     >

docs/install/kata-containers-3.0-rust-runtime-installation-guide.md

@@ -1,102 +0,0 @@
-# Kata Containers 3.0 rust runtime installation
-The following is an overview of the different installation methods available. 
-
-## Prerequisites
-
-Kata Containers 3.0 rust runtime requires nested virtualization or bare metal. Check 
-[hardware requirements](/src/runtime/README.md#hardware-requirements) to see if your system is capable of running Kata 
-Containers.
-
-### Platform support
-
-Kata Containers 3.0 rust runtime currently runs on 64-bit systems supporting the following
-architectures:
-
-> **Notes:**
-> For other architectures, see https://github.com/kata-containers/kata-containers/issues/4320
-
-| Architecture | Virtualization technology |
-|-|-|
-| `x86_64`| [Intel](https://www.intel.com) VT-x |
-| `aarch64` ("`arm64`")| [ARM](https://www.arm.com) Hyp |
-
-## Packaged installation methods
-
-| Installation method                                  | Description                                                                                  | Automatic updates | Use case                                                                                      | Availability
-|------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|----------- |
-| [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | Yes |
-| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users. | No |                                                                   
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 | No |
-| [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 | No |
-| [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      | No |
-| [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.  | Yes |              
-
-### Kata Deploy Installation
-
-Follow the [`kata-deploy`](../../tools/packaging/kata-deploy/README.md).
-### Official packages
-`ToDo`
-### Snap Installation
-`ToDo`
-### Automatic Installation
-`ToDo`
-### Manual Installation
-`ToDo`
-
-## Build from source installation
-
-### Rust Environment Set Up
-
-* Download `Rustup` and install  `Rust`
-    > **Notes:**
-    > Rust version 1.58 is needed
-
-    Example for `x86_64`
-    ```
-    $ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
-    $ source $HOME/.cargo/env
-    $ rustup install 1.58
-    $ rustup default 1.58-x86_64-unknown-linux-gnu
-    ```
-
-* Musl support for fully static binary
-    
-    Example for `x86_64`
-    ```
-    $ rustup target add x86_64-unknown-linux-musl
-    ```
-* [Musl `libc`](http://musl.libc.org/) install
-
-    Example for musl 1.2.3
-    ```
-    $ curl -O https://git.musl-libc.org/cgit/musl/snapshot/musl-1.2.3.tar.gz
-    $ tar vxf musl-1.2.3.tar.gz
-    $ cd musl-1.2.3/
-    $ ./configure --prefix=/usr/local/
-    $ make && sudo make install
-    ```
-
-
-### Install Kata 3.0 Rust Runtime Shim
-
-```
-$ git clone https://github.com/kata-containers/kata-containers.git
-$ cd kata-containers/src/runtime-rs
-$ make && sudo make install
-```
-After running the command above, the default config file `configuration.toml` will be installed under `/usr/share/defaults/kata-containers/`,  the binary file `containerd-shim-kata-v2` will be installed under `/usr/local/bin/` .
-
-### Build Kata Containers Kernel
-Follow the [Kernel installation guide](/tools/packaging/kernel/README.md).
-
-### Build Kata Rootfs
-Follow the [Rootfs installation guide](../../tools/osbuilder/rootfs-builder/README.md).
-
-### Build Kata Image
-Follow the [Image installation guide](../../tools/osbuilder/image-builder/README.md).
-
-### Install Containerd
-
-Follow the [Containerd installation guide](container-manager/containerd/containerd-install.md).
-
-

docs/install/minikube-installation-guide.md

@@ -55,11 +55,11 @@ Here are the features to set up a CRI-O based Minikube, and why you need them:
 
 | what | why |
 | ---- | --- |
-| `--bootstrapper=kubeadm` | As recommended for [minikube CRI-O](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) |
+| `--bootstrapper=kubeadm` | As recommended for [minikube CRI-o](https://kubernetes.io/docs/setup/minikube/#cri-o) |
 | `--container-runtime=cri-o` | Using CRI-O for Kata |
-| `--enable-default-cni` | As recommended for [minikube CRI-O](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) |
+| `--enable-default-cni` | As recommended for [minikube CRI-o](https://kubernetes.io/docs/setup/minikube/#cri-o) |
 | `--memory 6144` | Allocate sufficient memory, as Kata Containers default to 1 or 2Gb |
-| `--network-plugin=cni` | As recommended for [minikube CRI-O](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) |
+| `--network-plugin=cni` | As recommended for [minikube CRI-o](https://kubernetes.io/docs/setup/minikube/#cri-o) |
 | `--vm-driver kvm2` | The host VM driver |
 
 To use containerd, modify the `--container-runtime` argument:

docs/use-cases/GPU-passthrough-and-Kata.md

@@ -3,4 +3,4 @@
 Kata Containers supports passing certain GPUs from the host into the container. Select the GPU vendor for detailed information:
 
 - [Intel](Intel-GPU-passthrough-and-Kata.md)
-- [NVIDIA](NVIDIA-GPU-passthrough-and-Kata.md)
+- [Nvidia](Nvidia-GPU-passthrough-and-Kata.md)

docs/use-cases/NVIDIA-GPU-passthrough-and-Kata.md

@@ -1,592 +0,0 @@
-# Using NVIDIA GPU device with Kata Containers
-
-An NVIDIA GPU device can be passed to a Kata Containers container using GPU
-passthrough (NVIDIA GPU pass-through mode) as well as GPU mediated passthrough
-(NVIDIA `vGPU` mode).
-
-NVIDIA GPU pass-through mode, an entire physical GPU is directly assigned to one
-VM, bypassing the NVIDIA Virtual GPU Manager. In this mode of operation, the GPU
-is accessed exclusively by the NVIDIA driver running in the VM to which it is
-assigned. The GPU is not shared among VMs.
-
-NVIDIA Virtual GPU (`vGPU`) enables multiple virtual machines (VMs) to have
-simultaneous, direct access to a single physical GPU, using the same NVIDIA
-graphics drivers that are deployed on non-virtualized operating systems. By
-doing this, NVIDIA `vGPU` provides VMs with unparalleled graphics performance,
-compute performance, and application compatibility, together with the
-cost-effectiveness and scalability brought about by sharing a GPU among multiple
-workloads. A `vGPU` can be either time-sliced or Multi-Instance GPU (MIG)-backed
-with [MIG-slices](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/).
-
-| Technology | Description | Behavior | Detail |
-| --- | --- | --- | --- |
-| NVIDIA GPU pass-through mode | GPU passthrough | Physical GPU assigned to a single VM | Direct GPU assignment to VM without limitation |
-| NVIDIA vGPU time-sliced | GPU time-sliced | Physical GPU time-sliced for multiple VMs | Mediated passthrough |
-| NVIDIA vGPU MIG-backed | GPU with MIG-slices | Physical GPU MIG-sliced for multiple VMs | Mediated passthrough |
-
-## Hardware Requirements
-
-NVIDIA GPUs Recommended for Virtualization:
-
-- NVIDIA Tesla (T4, M10, P6, V100 or newer)
-- NVIDIA Quadro RTX 6000/8000
-
-## Host BIOS Requirements
-
-Some hardware requires a larger PCI BARs window, for example, NVIDIA Tesla P100,
-K40m
-
-```sh
-$ lspci -s d0:00.0 -vv | grep Region
-        Region 0: Memory at e7000000 (32-bit, non-prefetchable) [size=16M]
-        Region 1: Memory at 222800000000 (64-bit, prefetchable) [size=32G] # Above 4G
-        Region 3: Memory at 223810000000 (64-bit, prefetchable) [size=32M]
-```
-
-For large BARs devices, MMIO mapping above 4G address space should be `enabled`
-in the PCI configuration of the BIOS.
-
-Some hardware vendors use a different name in BIOS, such as:
-
-- Above 4G Decoding
-- Memory Hole for PCI MMIO
-- Memory Mapped I/O above 4GB
-
-If one is using a GPU based on the Ampere architecture and later additionally
-SR-IOV needs to be enabled for the `vGPU` use-case.
-
-The following steps outline the workflow for using an NVIDIA GPU with Kata.
-
-## Host Kernel Requirements
-
-The following configurations need to be enabled on your host kernel:
-
-- `CONFIG_VFIO`
-- `CONFIG_VFIO_IOMMU_TYPE1`
-- `CONFIG_VFIO_MDEV`
-- `CONFIG_VFIO_MDEV_DEVICE`
-- `CONFIG_VFIO_PCI`
-
-Your host kernel needs to be booted with `intel_iommu=on` on the kernel command
-line.
-
-## Install and configure Kata Containers
-
-To use non-large BARs devices (for example, NVIDIA Tesla T4), you need Kata
-version 1.3.0 or above. Follow the [Kata Containers setup
-instructions](../install/README.md) to install the latest version of Kata.
-
-To use large BARs devices (for example, NVIDIA Tesla P100), you need Kata
-version 1.11.0 or above.
-
-The following configuration in the Kata `configuration.toml` file as shown below
-can work:
-
-Hotplug for PCI devices with small BARs by `acpi_pcihp` (Linux's ACPI PCI
-Hotplug driver):
-
-```sh
-machine_type = "q35"
-
-hotplug_vfio_on_root_bus = false
-```
-
-Hotplug for PCIe devices with large BARs by `pciehp` (Linux's PCIe Hotplug
-driver):
-
-```sh
-machine_type = "q35"
-
-hotplug_vfio_on_root_bus = true
-pcie_root_port = 1
-```
-
-## Build Kata Containers kernel with GPU support
-
-The default guest kernel installed with Kata Containers does not provide GPU
-support. To use an NVIDIA GPU with Kata Containers, you need to build a kernel
-with the necessary GPU support.
-
-The following kernel config options need to be enabled:
-
-```sh
-# Support PCI/PCIe device hotplug (Required for large BARs device)
-CONFIG_HOTPLUG_PCI_PCIE=y
-
-# Support for loading modules (Required for load NVIDIA drivers)
-CONFIG_MODULES=y
-CONFIG_MODULE_UNLOAD=y
-
-# Enable the MMIO access method for PCIe devices (Required for large BARs device)
- CONFIG_PCI_MMCONFIG=y
-```
-
-The following kernel config options need to be disabled:
-
-```sh
-# Disable Open Source NVIDIA driver nouveau
-# It conflicts with NVIDIA official driver
-CONFIG_DRM_NOUVEAU=n
-```
-
-> **Note**: `CONFIG_DRM_NOUVEAU` is normally disabled by default.
-It is worth checking that it is not enabled in your kernel configuration to
-prevent any conflicts.
-
-Build the Kata Containers kernel with the previous config options, using the
-instructions described in [Building Kata Containers
-kernel](../../tools/packaging/kernel). For further details on building and
-installing guest kernels, see [the developer
-guide](../Developer-Guide.md#install-guest-kernel-images).
-
-There is an easy way to build a guest kernel that supports NVIDIA GPU:
-
-```sh
-## Build guest kernel with ../../tools/packaging/kernel
-
-# Prepare (download guest kernel source, generate .config)
-$ ./build-kernel.sh -v 5.15.23 -g nvidia -f setup
-
-# Build guest kernel
-$ ./build-kernel.sh -v 5.15.23 -g nvidia build
-
-# Install guest kernel
-$ sudo -E ./build-kernel.sh -v 5.15.23 -g nvidia install
-```
-
-To build NVIDIA Driver in Kata container, `linux-headers` are required.
-This is a way to generate deb packages for `linux-headers`:
-
-> **Note**:
-> Run `make rpm-pkg` to build the rpm package.
-> Run `make deb-pkg` to build the deb package.
->
-
-```sh
-$ cd kata-linux-5.15.23-89
-$ make deb-pkg
-```
-Before using the new guest kernel, please update the `kernel` parameters in
- `configuration.toml`.
-
-```sh
-kernel = "/usr/share/kata-containers/vmlinuz-nvidia-gpu.container"
-```
-
-## NVIDIA GPU pass-through mode with Kata Containers
-
-Use the following steps to pass an NVIDIA GPU device in pass-through mode with Kata:
-
-1. Find the Bus-Device-Function (BDF) for the GPU device on the host:
-
-   ```sh
-   $ sudo lspci -nn -D | grep -i nvidia
-   0000:d0:00.0 3D controller [0302]: NVIDIA Corporation Device [10de:20b9] (rev a1)
-   ```
-
-   > PCI address `0000:d0:00.0` is assigned to the hardware GPU device.
-   > `10de:20b9` is the device ID of the hardware GPU device.
-
-2. Find the IOMMU group for the GPU device:
-
-   ```sh
-   $ BDF="0000:d0:00.0"
-   $ readlink -e /sys/bus/pci/devices/$BDF/iommu_group
-   ```
-
-   The previous output shows that the GPU belongs to IOMMU group 192. The next
-   step is to bind the GPU to the VFIO-PCI driver.
-
-   ```sh
-   $ BDF="0000:d0:00.0"
-   $ DEV="/sys/bus/pci/devices/$BDF"
-   $ echo "vfio-pci" > $DEV/driver_override
-   $ echo $BDF > $DEV/driver/unbind
-   $ echo $BDF > /sys/bus/pci/drivers_probe
-   # To return the device to the standard driver, we simply clear the
-   # driver_override and reprobe the device, ex:
-   $ echo > $DEV/preferred_driver
-   $ echo $BDF > $DEV/driver/unbind
-   $ echo $BDF > /sys/bus/pci/drivers_probe
-   ```
-
-3. Check the IOMMU group number under `/dev/vfio`:
-
-   ```sh
-   $ ls -l /dev/vfio
-   total 0
-   crw------- 1 zvonkok zvonkok 243,   0 Mar 18 03:06 192
-   crw-rw-rw- 1 root    root     10, 196 Mar 18 02:27 vfio
-   ```
-
-4. Start a Kata container with the GPU device:
-
-   ```sh
-   # You may need to `modprobe vhost-vsock` if you get
-   # host system doesn't support vsock: stat /dev/vhost-vsock
-   $ sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t  "docker.io/library/archlinux:latest" arch uname -r
-   ```
-
-5. Run `lspci` within the container to verify the GPU device is seen in the list
-   of the PCI devices. Note the vendor-device id of the GPU (`10de:20b9`) in the `lspci` output.
-
-   ```sh
-   $ sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t  "docker.io/library/archlinux:latest" arch sh -c "lspci -nn | grep '10de:20b9'"
-   ```
-
-6. Additionally, you can check the PCI BARs space of ​​the NVIDIA GPU device in the container:
-
-   ```sh
-   $ sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t  "docker.io/library/archlinux:latest" arch sh -c "lspci -s 02:00.0 -vv | grep Region"
-   ```
-
-   > **Note**: If you see a message similar to the above, the BAR space of the NVIDIA
-   > GPU has been successfully allocated.
-
-## NVIDIA vGPU mode with Kata Containers
-
-NVIDIA vGPU is a licensed product on all supported GPU boards. A software license
-is required to enable all vGPU features within the guest VM. NVIDIA vGPU manager
-needs to be installed on the host to configure GPUs in vGPU mode. See [NVIDIA Virtual GPU Software Documentation v14.0 through 14.1](https://docs.nvidia.com/grid/14.0/) for more details.
-
-### NVIDIA vGPU time-sliced
-
-In the time-sliced mode, the GPU is not partitioned and the workload uses the
-whole GPU and shares access to the GPU engines. Processes are scheduled in
-series. The best effort scheduler is the default one and can be exchanged by
-other scheduling policies see the documentation above how to do that.
-
-Beware if you had `MIG` enabled before to disable `MIG` on the GPU if you want
-to use `time-sliced` `vGPU`.
-
-```sh
-$ sudo nvidia-smi -mig 0
-```
-
-Enable the virtual functions for the physical GPU in the `sysfs` file system.
-
-```sh
-$ sudo /usr/lib/nvidia/sriov-manage -e 0000:41:00.0
-```
-
-Get the `BDF` of the available virtual function on the GPU, and choose one for the
-following steps.
-
-```sh
-$ cd /sys/bus/pci/devices/0000:41:00.0/
-$ ls -l |  grep virtfn
-```
-
-#### List all available vGPU instances
-
-The following shell snippet will walk the `sysfs` and only print instances
-that are available, that can be created.
-
-```sh
-# The 00.0 is often the PF of the device the VFs will have the funciont in the
-# BDF incremented by some values so e.g. the very first VF is 0000:41:00.4
-
-cd /sys/bus/pci/devices/0000:41:00.0/
-
-for vf in $(ls -d virtfn*)
-do
-        BDF=$(basename $(readlink -f $vf))
-        for md in $(ls -d $vf/mdev_supported_types/*)
-        do
-                AVAIL=$(cat $md/available_instances)
-                NAME=$(cat $md/name)
-                DIR=$(basename $md)
-
-                if [ $AVAIL -gt 0 ]; then
-                        echo "| BDF          | INSTANCES | NAME           | DIR        |"
-                        echo "+--------------+-----------+----------------+------------+"
-                        printf "| %12s |%10d |%15s | %10s |\n\n" "$BDF" "$AVAIL" "$NAME" "$DIR"
-                fi
-
-        done
-done
-```
-
-If there are available instances you get something like this (for the first VF),
-beware that the output is highly dependent on the GPU you have, if there is no
-output check again if `MIG` is really disabled.
-
-```sh
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 |  GRID A100D-4C | nvidia-692 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 |  GRID A100D-8C | nvidia-693 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 | GRID A100D-10C | nvidia-694 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 | GRID A100D-16C | nvidia-695 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 | GRID A100D-20C | nvidia-696 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 | GRID A100D-40C | nvidia-697 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 | GRID A100D-80C | nvidia-698 |
-
-```
-
-Change to the `mdev_supported_types` directory for the virtual function on which
-you want to create the `vGPU`. Taking the first output as an example:
-
-```sh
-$ cd virtfn0/mdev_supported_types/nvidia-692
-$ UUIDGEN=$(uuidgen)
-$ sudo bash -c "echo $UUIDGEN > create"
-```
-
-Confirm that the `vGPU` was created. You should see the `UUID` pointing to a
-subdirectory of the `sysfs` space.
-
-```sh
-$ ls -l /sys/bus/mdev/devices/
-```
-
-Get the `IOMMU` group number and verify there is a `VFIO` device created to use
-with Kata.
-
-```sh
-$ ls -l /sys/bus/mdev/devices/*/
-$ ls -l /dev/vfio
-```
-
-Use the `VFIO` device created in the same way as in the pass-through use-case.
-Beware that the guest needs the NVIDIA guest drivers, so one would need to build
-a new guest `OS` image.
-
-### NVIDIA vGPU MIG-backed
-
-We're not going into detail what `MIG` is but briefly it is a technology to
-partition the hardware into independent instances with guaranteed quality of
-service. For more details see [NVIDIA Multi-Instance GPU User Guide](https://docs.nvidia.com/datacenter/tesla/mig-user-guide/).
-
-First enable `MIG` mode for a GPU, depending on the platform you're running
-a reboot would be necessary. Some platforms support GPU reset.
-
-```sh
-$ sudo nvidia-smi -mig 1
-```
-
-If the platform supports a GPU reset one can run, otherwise you will get a
-warning to reboot the server.
-
-```sh
-$ sudo nvidia-smi --gpu-reset
-```
-
-The driver per default provides a number of profiles that users can opt-in when
-configuring the MIG feature.
-
-```sh
-$ sudo nvidia-smi mig -lgip
-+-----------------------------------------------------------------------------+
-| GPU instance profiles:                                                      |
-| GPU   Name             ID    Instances   Memory     P2P    SM    DEC   ENC  |
-|                              Free/Total   GiB              CE    JPEG  OFA  |
-|=============================================================================|
-|   0  MIG 1g.10gb       19     7/7        9.50       No     14     0     0   |
-|                                                             1     0     0   |
-+-----------------------------------------------------------------------------+
-|   0  MIG 1g.10gb+me    20     1/1        9.50       No     14     1     0   |
-|                                                             1     1     1   |
-+-----------------------------------------------------------------------------+
-|   0  MIG 2g.20gb       14     3/3        19.50      No     28     1     0   |
-|                                                             2     0     0   |
-+-----------------------------------------------------------------------------+
-                              ...
-```
-
-Create the GPU instances that correspond to the `vGPU` types of the `MIG-backed`
-`vGPUs` that you will create [NVIDIA A100 PCIe 80GB Virtual GPU Types](https://docs.nvidia.com/grid/13.0/grid-vgpu-user-guide/index.html#vgpu-types-nvidia-a100-pcie-80gb).
-
-```sh
-# MIG 1g.10gb --> vGPU A100D-1-10C
-$ sudo nvidia-smi mig -cgi 19
-```
-
-List the GPU instances and get the GPU instance id to create the compute
-instance.
-
-```sh
-$ sudo nvidia-smi mig -lgi # list the created GPU instances
-$ sudo nvidia-smi mig -cci -gi 9 # each GPU instance can have several compute
-                                 # instances. Instance -> Workload
-```
-
-Verify that the compute instances were created within the GPU instance
-
-```sh
-$ nvidia-smi
-                              ... snip ...
-+-----------------------------------------------------------------------------+
-| MIG devices:                                                                |
-+------------------+----------------------+-----------+-----------------------+
-| GPU  GI  CI  MIG |         Memory-Usage |        Vol|         Shared        |
-|      ID  ID  Dev |           BAR1-Usage | SM     Unc| CE  ENC  DEC  OFA  JPG|
-|                  |                      |        ECC|                       |
-|==================+======================+===========+=======================|
-|  0    9   0   0  |      0MiB /  9728MiB | 14      0 |  1   0    0    0    0 |
-|                  |      0MiB /  4095MiB |           |                       |
-+------------------+----------------------+-----------+-----------------------+
-                              ... snip ...
-```
-
-We can use the [snippet](#list-all-available-vgpu-instances) from before to list
-the available `vGPU` instances, this time `MIG-backed`.
-
-```sh
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.4 |         1 |GRID A100D-1-10C | nvidia-699 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:00.5 |         1 |GRID A100D-1-10C | nvidia-699 |
-
-| BDF          | INSTANCES | NAME           | DIR        |
-+--------------+-----------+----------------+------------+
-| 0000:41:01.6 |         1 |GRID A100D-1-10C | nvidia-699 |
-                       ... snip ...
-```
-
-Repeat the steps after the [snippet](#list-all-available-vgpu-instances) listing
-to create the corresponding `mdev` device and use the guest `OS` created in the
-previous section with `time-sliced` `vGPUs`.
-
-## Install NVIDIA Driver + Toolkit in Kata Containers Guest OS
-
-Consult the [Developer-Guide](https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#create-a-rootfs-image) on how to create a
-rootfs base image for a distribution of your choice. This is going to be used as
-a base for a NVIDIA enabled guest OS. Use the `EXTRA_PKGS` variable to install
-all the needed packages to compile the drivers. Also copy the kernel development
-packages from the previous `make deb-pkg` into `$ROOTFS_DIR`.
-
-```sh
-export EXTRA_PKGS="gcc make curl gnupg"
-```
-
-Having the `$ROOTFS_DIR` exported in the previous step we can now install all the
-needed parts in the guest OS. In this case, we have an Ubuntu based rootfs.
-
-First off all mount the special filesystems into the rootfs
-
-```sh
-$ sudo mount -t sysfs -o ro none ${ROOTFS_DIR}/sys
-$ sudo mount -t proc -o ro none ${ROOTFS_DIR}/proc
-$ sudo mount -t tmpfs none ${ROOTFS_DIR}/tmp
-$ sudo mount -o bind,ro /dev ${ROOTFS_DIR}/dev
-$ sudo mount -t devpts none ${ROOTFS_DIR}/dev/pts
-```
-
-Now we can enter `chroot`
-
-```sh
-$ sudo chroot ${ROOTFS_DIR}
-```
-
-Inside the rootfs one is going to install the drivers and toolkit to enable the
-easy creation of GPU containers with Kata. We can also use this rootfs for any
-other container not specifically only for GPUs.
-
-As a prerequisite install the copied kernel development packages
-
-```sh
-$ sudo dpkg -i *.deb
-```
-
-Get the driver run file, since we need to build the driver against a kernel that
-is not running on the host we need the ability to specify the exact version we
-want the driver to build against. Take the kernel version one used for building
-the NVIDIA kernel (`5.15.23-nvidia-gpu`).
-
-```sh
-$ wget https://us.download.nvidia.com/XFree86/Linux-x86_64/510.54/NVIDIA-Linux-x86_64-510.54.run
-$ chmod +x NVIDIA-Linux-x86_64-510.54.run
-# Extract the source files so we can run the installer with arguments
-$ ./NVIDIA-Linux-x86_64-510.54.run -x
-$ cd NVIDIA-Linux-x86_64-510.54
-$ ./nvidia-installer -k 5.15.23-nvidia-gpu
-```
-
-Having the drivers installed we need to install the toolkit which will take care
-of providing the right bits into the container.
-
-```sh
-$ distribution=$(. /etc/os-release;echo $ID$VERSION_ID)
-$ curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
-$ curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
-$ apt update
-$ apt install nvidia-container-toolkit
-```
-
-Create the hook execution file for Kata:
-
-```
-# Content of $ROOTFS_DIR/usr/share/oci/hooks/prestart/nvidia-container-toolkit.sh
-
-#!/bin/bash -x
-
-/usr/bin/nvidia-container-toolkit -debug $@
-```
-
-As the last step one can do some cleanup of files or package caches. Build the
-rootfs and configure it for use with Kata according to the development guide.
-
-Enable the `guest_hook_path` in Kata's `configuration.toml`
-
-```sh
-guest_hook_path = "/usr/share/oci/hooks"
-```
-
-One has built a NVIDIA rootfs, kernel and now we can run any GPU container
-without installing the drivers into the container. Check NVIDIA device status
-with `nvidia-smi`
-
-```sh
-$  sudo ctr --debug run --runtime "io.containerd.kata.v2"  --device /dev/vfio/192  --rm -t "docker.io/nvidia/cuda:11.6.0-base-ubuntu20.04" cuda nvidia-smi
-Fri Mar 18 10:36:59 2022
-+-----------------------------------------------------------------------------+
-| NVIDIA-SMI 510.54       Driver Version: 510.54       CUDA Version: 11.6     |
-|-------------------------------+----------------------+----------------------+
-| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
-| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
-|                               |                      |               MIG M. |
-|===============================+======================+======================|
-|   0  NVIDIA A30X         Off  | 00000000:02:00.0 Off |                    0 |
-| N/A   38C    P0    67W / 230W |      0MiB / 24576MiB |      0%      Default |
-|                               |                      |             Disabled |
-+-------------------------------+----------------------+----------------------+
-
-+-----------------------------------------------------------------------------+
-| Processes:                                                                  |
-|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |
-|        ID   ID                                                   Usage      |
-|=============================================================================|
-|  No running processes found                                                 |
-+-----------------------------------------------------------------------------+
-```
-
-As the last step one can remove the additional packages and files that were added
-to the `$ROOTFS_DIR` to keep it as small as possible.
-
-## References
-
-- [Configuring a VM for GPU Pass-Through by Using the QEMU Command Line](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/index.html#using-gpu-pass-through-red-hat-el-qemu-cli)
-- https://gitlab.com/nvidia/container-images/driver/-/tree/master
-- https://github.com/NVIDIA/nvidia-docker/wiki/Driver-containers

docs/use-cases/Nvidia-GPU-passthrough-and-Kata.md

@@ -0,0 +1,293 @@
+# Using Nvidia GPU device with Kata Containers
+
+An Nvidia GPU device can be passed to a Kata Containers container using GPU passthrough
+(Nvidia GPU pass-through mode) as well as GPU mediated passthrough (Nvidia vGPU mode). 
+
+Nvidia GPU pass-through mode, an entire physical GPU is directly assigned to one VM,
+bypassing the Nvidia Virtual GPU Manager. In this mode of operation, the GPU is accessed
+exclusively by the Nvidia driver running in the VM to which it is assigned.
+The GPU is not shared among VMs.
+
+Nvidia Virtual GPU (vGPU) enables multiple virtual machines (VMs) to have simultaneous,
+direct access to a single physical GPU, using the same Nvidia graphics drivers that are
+deployed on non-virtualized operating systems. By doing this, Nvidia vGPU provides VMs
+with unparalleled graphics performance, compute performance, and application compatibility,
+together with the cost-effectiveness and scalability brought about by sharing a GPU
+among multiple workloads.
+
+| Technology | Description | Behaviour | Detail |
+| --- | --- | --- | --- |
+| Nvidia GPU pass-through mode | GPU passthrough | Physical GPU assigned to a single VM | Direct GPU assignment to VM without limitation |
+| Nvidia vGPU mode | GPU sharing | Physical GPU shared by multiple VMs | Mediated passthrough |
+
+## Hardware Requirements
+Nvidia GPUs Recommended for Virtualization:
+
+- Nvidia Tesla (T4, M10, P6, V100 or newer)
+- Nvidia Quadro RTX 6000/8000
+
+## Host BIOS Requirements
+
+Some hardware requires a larger PCI BARs window, for example, Nvidia Tesla P100, K40m
+```
+$ lspci -s 04:00.0 -vv | grep Region
+      Region 0: Memory at c6000000 (32-bit, non-prefetchable) [size=16M]
+      Region 1: Memory at 383800000000 (64-bit, prefetchable) [size=16G] #above 4G
+      Region 3: Memory at 383c00000000 (64-bit, prefetchable) [size=32M]
+```
+
+For large BARs devices, MMIO mapping above 4G address space should be `enabled`
+in the PCI configuration of the BIOS.
+
+Some hardware vendors use different name in BIOS, such as:
+
+- Above 4G Decoding
+- Memory Hole for PCI MMIO
+- Memory Mapped I/O above 4GB
+
+The following steps outline the workflow for using an Nvidia GPU with Kata.
+
+## Host Kernel Requirements
+The following configurations need to be enabled on your host kernel:
+
+- `CONFIG_VFIO`
+- `CONFIG_VFIO_IOMMU_TYPE1`
+- `CONFIG_VFIO_MDEV`
+- `CONFIG_VFIO_MDEV_DEVICE`
+- `CONFIG_VFIO_PCI`
+
+Your host kernel needs to be booted with `intel_iommu=on` on the kernel command line.
+
+## Install and configure Kata Containers
+To use non-large BARs devices (for example, Nvidia Tesla T4), you need Kata version 1.3.0 or above.
+Follow the [Kata Containers setup instructions](../install/README.md)
+to install the latest version of Kata.
+
+To use large BARs devices (for example, Nvidia Tesla P100), you need Kata version 1.11.0 or above.
+
+The following configuration in the Kata `configuration.toml` file as shown below can work:
+
+Hotplug for PCI devices by `acpi_pcihp` (Linux's ACPI PCI Hotplug driver):
+```
+machine_type = "q35"
+
+hotplug_vfio_on_root_bus = false
+```
+
+Hotplug for PCIe devices by `pciehp` (Linux's PCIe Hotplug driver):
+```
+machine_type = "q35"
+
+hotplug_vfio_on_root_bus = true
+pcie_root_port = 1
+```
+
+## Build Kata Containers kernel with GPU support
+The default guest kernel installed with Kata Containers does not provide GPU support.
+To use an Nvidia GPU with Kata Containers, you need to build a kernel with the
+necessary GPU support.
+
+The following kernel config options need to be enabled:
+```
+# Support PCI/PCIe device hotplug (Required for large BARs device)
+CONFIG_HOTPLUG_PCI_PCIE=y
+
+# Support for loading modules (Required for load Nvidia drivers)
+CONFIG_MODULES=y
+CONFIG_MODULE_UNLOAD=y
+
+# Enable the MMIO access method for PCIe devices (Required for large BARs device)
+CONFIG_PCI_MMCONFIG=y
+```
+
+The following kernel config options need to be disabled:
+```
+# Disable Open Source Nvidia driver nouveau
+# It conflicts with Nvidia official driver
+CONFIG_DRM_NOUVEAU=n
+```
+> **Note**: `CONFIG_DRM_NOUVEAU` is normally disabled by default.
+It is worth checking that it is not enabled in your kernel configuration to prevent any conflicts.
+
+
+Build the Kata Containers kernel with the previous config options,
+using the instructions described in [Building Kata Containers kernel](../../tools/packaging/kernel).
+For further details on building and installing guest kernels,
+see [the developer guide](../Developer-Guide.md#install-guest-kernel-images).
+
+There is an easy way to build a guest kernel that supports Nvidia GPU:
+```
+## Build guest kernel with ../../tools/packaging/kernel
+
+# Prepare (download guest kernel source, generate .config)
+$ ./build-kernel.sh -v 4.19.86 -g nvidia -f setup
+
+# Build guest kernel
+$ ./build-kernel.sh -v 4.19.86 -g nvidia build
+
+# Install guest kernel
+$ sudo -E ./build-kernel.sh -v 4.19.86 -g nvidia install
+/usr/share/kata-containers/vmlinux-nvidia-gpu.container -> vmlinux-4.19.86-70-nvidia-gpu
+/usr/share/kata-containers/vmlinuz-nvidia-gpu.container -> vmlinuz-4.19.86-70-nvidia-gpu
+```
+
+To build Nvidia Driver in Kata container, `kernel-devel` is required.  
+This is a way to generate rpm packages for `kernel-devel`:
+```
+$ cd kata-linux-4.19.86-68
+$ make rpm-pkg
+Output RPMs:
+~/rpmbuild/RPMS/x86_64/kernel-devel-4.19.86_nvidia_gpu-1.x86_64.rpm
+```
+> **Note**:
+> - `kernel-devel` should be installed in Kata container before run Nvidia driver installer.
+> - Run `make deb-pkg` to build the deb package.
+
+Before using the new guest kernel, please update the `kernel` parameters in `configuration.toml`.
+```
+kernel = "/usr/share/kata-containers/vmlinuz-nvidia-gpu.container"
+```
+
+## Nvidia GPU pass-through mode with Kata Containers
+Use the following steps to pass an Nvidia GPU device in pass-through mode with Kata:
+
+1. Find the Bus-Device-Function (BDF) for GPU device on host:
+   ```
+   $ sudo lspci -nn -D | grep -i nvidia
+   0000:04:00.0 3D controller [0302]: NVIDIA Corporation Device [10de:15f8] (rev a1)
+   0000:84:00.0 3D controller [0302]: NVIDIA Corporation Device [10de:15f8] (rev a1)
+   ```
+   > PCI address `0000:04:00.0` is assigned to the hardware GPU device.
+   > `10de:15f8` is the device ID of the hardware GPU device.
+
+2. Find the IOMMU group for the GPU device:
+   ```
+   $ BDF="0000:04:00.0"
+   $ readlink -e /sys/bus/pci/devices/$BDF/iommu_group
+   /sys/kernel/iommu_groups/45
+   ```
+   The previous output shows that the GPU belongs to IOMMU group 45.
+
+3. Check the IOMMU group number under `/dev/vfio`:
+   ```
+   $ ls -l /dev/vfio
+   total 0
+   crw------- 1 root root 248,   0 Feb 28 09:57 45
+   crw------- 1 root root 248,   1 Feb 28 09:57 54
+   crw-rw-rw- 1 root root  10, 196 Feb 28 09:57 vfio
+   ```
+
+4. Start a Kata container with GPU device:
+   ```
+   $ sudo docker run -it --runtime=kata-runtime --cap-add=ALL --device /dev/vfio/45 centos /bin/bash
+   ```
+
+5. Run `lspci` within the container to verify the GPU device is seen in the list
+   of the PCI devices. Note the vendor-device id of the GPU (`10de:15f8`) in the `lspci` output.
+   ```
+   $ lspci -nn -D | grep '10de:15f8'
+   0000:01:01.0 3D controller [0302]: NVIDIA Corporation GP100GL [Tesla P100 PCIe 16GB] [10de:15f8] (rev a1)
+   ```
+
+6. Additionally, you can check the PCI BARs space of ​​the Nvidia GPU device in the container:
+   ```
+   $ lspci -s 01:01.0 -vv | grep Region
+		Region 0: Memory at c0000000 (32-bit, non-prefetchable) [disabled] [size=16M]
+		Region 1: Memory at 4400000000 (64-bit, prefetchable) [disabled] [size=16G]
+		Region 3: Memory at 4800000000 (64-bit, prefetchable) [disabled] [size=32M]
+   ```
+   > **Note**: If you see a message similar to the above, the BAR space of the Nvidia
+   > GPU has been successfully allocated.
+
+## Nvidia vGPU mode with Kata Containers
+
+Nvidia vGPU is a licensed product on all supported GPU boards. A software license
+is required to enable all vGPU features within the guest VM.
+
+> **Note**: There is no suitable test environment, so it is not written here.
+
+
+## Install Nvidia Driver in Kata Containers
+Download the official Nvidia driver from
+[https://www.nvidia.com/Download/index.aspx](https://www.nvidia.com/Download/index.aspx),
+for example `NVIDIA-Linux-x86_64-418.87.01.run`.
+
+Install the `kernel-devel`(generated in the previous steps) for guest kernel:
+```
+$ sudo rpm -ivh kernel-devel-4.19.86_gpu-1.x86_64.rpm
+```
+
+Here is an example to extract, compile and install Nvidia driver:
+```
+## Extract
+$ sh ./NVIDIA-Linux-x86_64-418.87.01.run -x
+
+## Compile and install (It will take some time)
+$ cd NVIDIA-Linux-x86_64-418.87.01
+$ sudo ./nvidia-installer -a -q --ui=none \
+ --no-cc-version-check \
+ --no-opengl-files --no-install-libglvnd \
+ --kernel-source-path=/usr/src/kernels/`uname -r`
+```
+
+Or just run one command line:
+```
+$ sudo sh ./NVIDIA-Linux-x86_64-418.87.01.run -a -q --ui=none \
+ --no-cc-version-check \
+ --no-opengl-files --no-install-libglvnd \
+ --kernel-source-path=/usr/src/kernels/`uname -r`
+```
+
+To view detailed logs of the installer:
+```
+$ tail -f /var/log/nvidia-installer.log
+```
+
+Load Nvidia driver module manually
+```
+# Optional（generate modules.dep and map files for Nvidia driver）
+$ sudo depmod
+
+# Load module
+$ sudo modprobe nvidia-drm
+
+# Check module
+$ lsmod | grep nvidia
+nvidia_drm             45056  0
+nvidia_modeset       1093632  1 nvidia_drm
+nvidia              18202624  1 nvidia_modeset
+drm_kms_helper        159744  1 nvidia_drm
+drm                   364544  3 nvidia_drm,drm_kms_helper
+i2c_core               65536  3 nvidia,drm_kms_helper,drm
+ipmi_msghandler        49152  1 nvidia
+```
+
+
+Check Nvidia device status with `nvidia-smi`
+```
+$ nvidia-smi
+Tue Mar  3 00:03:49 2020
++-----------------------------------------------------------------------------+
+| NVIDIA-SMI 418.87.01    Driver Version: 418.87.01    CUDA Version: 10.1     |
+|-------------------------------+----------------------+----------------------+
+| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |
+| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |
+|===============================+======================+======================|
+|   0  Tesla P100-PCIE...  Off  | 00000000:01:01.0 Off |                    0 |
+| N/A   27C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |
++-------------------------------+----------------------+----------------------+
+
++-----------------------------------------------------------------------------+
+| Processes:                                                       GPU Memory |
+|  GPU       PID   Type   Process name                             Usage      |
+|=============================================================================|
+|  No running processes found                                                 |
++-----------------------------------------------------------------------------+
+
+```
+
+## References
+
+- [Configuring a VM for GPU Pass-Through by Using the QEMU Command Line](https://docs.nvidia.com/grid/latest/grid-vgpu-user-guide/index.html#using-gpu-pass-through-red-hat-el-qemu-cli)
+- https://gitlab.com/nvidia/container-images/driver/-/tree/master
+- https://github.com/NVIDIA/nvidia-docker/wiki/Driver-containers

docs/use-cases/using-Intel-QAT-and-kata.md

@@ -279,8 +279,8 @@ $ export KERNEL_EXTRAVERSION=$(awk '/^EXTRAVERSION =/{print $NF}' $GOPATH/$LINUX
 $ export KERNEL_ROOTFS_DIR=${KERNEL_MAJOR_VERSION}.${KERNEL_PATHLEVEL}.${KERNEL_SUBLEVEL}${KERNEL_EXTRAVERSION}
 $ cd $QAT_SRC
 $ KERNEL_SOURCE_ROOT=$GOPATH/$LINUX_VER ./configure --enable-icp-sriov=guest
-$ sudo -E make all -j $($(nproc ${CI:+--ignore 1}))
-$ sudo -E make INSTALL_MOD_PATH=$ROOTFS_DIR qat-driver-install -j $($(nproc ${CI:+--ignore 1}))
+$ sudo -E make all -j$(nproc)
+$ sudo -E make INSTALL_MOD_PATH=$ROOTFS_DIR qat-driver-install -j$(nproc)
 ```
 
 The `usdm_drv` module also needs to be copied into the rootfs modules path and
@@ -312,7 +312,7 @@ working properly with the Kata Containers VM.
 
 ### Build OpenSSL Intel® QAT engine container
 
-Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/demo/openssl-qat-engine) 
+Use the OpenSSL Intel® QAT [Dockerfile](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/master/demo/openssl-qat-engine) 
 to build a container image with an optimized OpenSSL engine for 
 Intel® QAT. Using `docker build` with the Kata Containers runtime can sometimes
 have issues. Therefore, make sure that `runc` is the default Docker container 
@@ -444,7 +444,7 @@ $ sudo docker save -o openssl-qat-engine.tar openssl-qat-engine:latest
 $ sudo ctr -n=k8s.io images import openssl-qat-engine.tar
 ```
 
-The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/cmd/qat_plugin/README.md)
+The [Intel® QAT Plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/master/cmd/qat_plugin/README.md)
 needs to be started so that the virtual functions can be discovered and
 used by Kubernetes. 
 

docs/use-cases/using-Intel-SGX-and-kata.md

@@ -18,13 +18,16 @@ CONFIG_X86_SGX_KVM=y
 
 * Kubernetes cluster configured with:
    * [`kata-deploy`](../../tools/packaging/kata-deploy) based Kata Containers installation
-   * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images) and associated components including [operator](https://github.com/intel/intel-device-plugins-for-kubernetes/blob/main/cmd/operator/README.md) and dependencies
+   * [Intel SGX Kubernetes device plugin](https://github.com/intel/intel-device-plugins-for-kubernetes/tree/main/cmd/sgx_plugin#deploying-with-pre-built-images)
 
 > Note: Kata Containers supports creating VM sandboxes with Intel® SGX enabled
 > using [cloud-hypervisor](https://github.com/cloud-hypervisor/cloud-hypervisor/) and [QEMU](https://www.qemu.org/) VMMs only.
 
 ### Kata Containers Configuration
 
+Before running a Kata Container make sure that your version of `crio` or `containerd`
+supports annotations.
+
 For `containerd` check in `/etc/containerd/config.toml` that the list of `pod_annotations` passed
 to the `sandbox` are: `["io.katacontainers.*", "sgx.intel.com/epc"]`.
 
@@ -61,9 +64,6 @@ spec:
           name: eosgx-demo-job-1
           image: oeciteam/oe-helloworld:latest
           imagePullPolicy: IfNotPresent
-          volumeMounts:
-          - mountPath: /dev
-            name: dev-mount
           securityContext:
             readOnlyRootFilesystem: true
             capabilities:
@@ -99,4 +99,4 @@ because socket passthrough is not supported. An alternative is to deploy the `ae
 container.
 * Projects like [Gramine Shielded Containers (GSC)](https://gramine-gsc.readthedocs.io/en/latest/) are
 also known to work. For GSC specifically, the Kata guest kernel needs to have the `CONFIG_NUMA=y`
-enabled and at least one CPU online when running the GSC container. The Kata Containers guest kernel currently has `CONFIG_NUMA=y` enabled by default.
+enabled and at least one CPU online when running the GSC container.

docs/use-cases/using-vpp-and-kata.md

@@ -0,0 +1,76 @@
+# Setup to run VPP
+
+The Data Plane Development Kit (DPDK) is a set of libraries and drivers for
+fast packet processing. Vector Packet Processing (VPP) is a platform
+extensible framework that provides out-of-the-box production quality
+switch and router functionality. VPP is a high performance packet-processing
+stack that can run on commodity CPUs. Enabling VPP with DPDK support can
+yield significant performance improvements over a Linux\* bridge providing a
+switch with DPDK VHOST-USER ports.
+
+For more information about VPP visit their [wiki](https://wiki.fd.io/view/VPP).
+
+## Install and configure Kata Containers
+
+Follow  the [Kata Containers setup instructions](../Developer-Guide.md).
+
+In order to make use of VHOST-USER based interfaces, the container needs to be backed
+by huge pages. `HugePages` support is required for the large memory pool allocation used for
+DPDK packet buffers.  This is a feature which must be configured within the Linux Kernel. See
+[the DPDK documentation](https://doc.dpdk.org/guides/linux_gsg/sys_reqs.html#use-of-hugepages-in-the-linux-environment)
+for details on how to enable for the host. After enabling huge pages support on the host system,
+update the Kata configuration to enable huge page support in the guest kernel:
+
+```
+$ sudo sed -i -e 's/^# *\(enable_hugepages\).*=.*$/\1 = true/g' /usr/share/defaults/kata-containers/configuration.toml
+```
+
+
+## Install VPP
+
+Follow the [VPP installation instructions](https://wiki.fd.io/view/VPP/Installing_VPP_binaries_from_packages).
+
+After a successful installation, your host system is ready to start
+connecting Kata Containers with VPP bridges.
+
+### Install the VPP Docker\* plugin
+
+To create a Docker network and connect Kata Containers easily to that network through
+Docker, install a VPP Docker plugin.
+
+To install the plugin, follow the [plugin installation instructions](https://github.com/clearcontainers/vpp).
+
+This VPP plugin allows the creation of a VPP network. Every container added
+to this network is connected through an L2 bridge-domain provided by VPP.
+
+## Example: Launch two Kata Containers using VPP
+
+To use VPP, use Docker to create a network that makes use of VPP.
+For example:
+
+```
+$ sudo docker network create -d=vpp --ipam-driver=vpp --subnet=192.168.1.0/24 --gateway=192.168.1.1  vpp_net
+```
+
+Test connectivity by launching two containers:
+```
+$ sudo docker run --runtime=kata-runtime --net=vpp_net --ip=192.168.1.2 --mac-address=CA:FE:CA:FE:01:02 -it busybox bash -c "ip a; ip route; sleep 300"
+
+$ sudo docker run --runtime=kata-runtime --net=vpp_net --ip=192.168.1.3 --mac-address=CA:FE:CA:FE:01:03 -it busybox bash -c "ip a; ip route; ping 192.168.1.2"
+```
+
+These commands setup two Kata Containers connected via a VPP L2 bridge
+domain. The first of the two VMs displays the networking details and then
+sleeps providing a period of time for it to be pinged. The second
+VM displays its networking details and then pings the first VM, verifying
+connectivity between them.
+
+After verifying connectivity, cleanup with the following commands:
+
+```
+$ sudo docker kill $(sudo docker ps --no-trunc -aq)
+$ sudo docker rm $(sudo docker ps --no-trunc -aq)
+$ sudo docker network rm vpp_net
+$ sudo service vpp stop
+```
+

snap/README.md

@@ -22,35 +22,21 @@ $ sudo snap install kata-containers --classic
 
 ## Build and install snap image
 
-Run the command below which will use the packaging Makefile to build the snap image:
+Run next command at the root directory of the packaging repository.
 
 ```sh
-$ make -C tools/packaging snap
+$ make snap
 ```
 
-> **Warning:**
->
-> By default, `snapcraft` will create a clean virtual machine
-> environment to build the snap in using the `multipass` tool.
->
-> However, `multipass` is silently disabled when `--destructive-mode` is
-> used.
->
-> Since building the Kata Containers package currently requires
-> `--destructive-mode`, the snap will be built using the host
-> environment. To avoid parts of the build auto-detecting additional
-> features to enable (for example for QEMU), we recommend that you
-> only run the snap build in a minimal host environment.
-
 To install the resulting snap image, snap must be put in [classic mode][3] and the
-security confinement must be disabled (`--classic`). Also since the resulting snap
-has not been signed the verification of signature must be omitted (`--dangerous`).
+security confinement must be disabled (*--classic*). Also since the resulting snap
+has not been signed the verification of signature must be omitted (*--dangerous*).
 
 ```sh
-$ sudo snap install --classic --dangerous "kata-containers_${version}_${arch}.snap"
+$ sudo snap install --classic --dangerous kata-containers_[VERSION]_[ARCH].snap
 ```
 
-Replace `${version}` with the current version of Kata Containers and `${arch}` with
+Replace `VERSION` with the current version of Kata Containers and `ARCH` with
 the system architecture.
 
 ## Configure Kata Containers
@@ -90,12 +76,12 @@ then a new configuration file can be [created](#configure-kata-containers)
 and [configured][7].
 
 [1]: https://docs.snapcraft.io/snaps/intro
-[2]: ../../docs/design/architecture/README.md#root-filesystem-image
+[2]: ../docs/design/architecture/README.md#root-filesystem-image
 [3]: https://docs.snapcraft.io/reference/confinement#classic
-[4]: https://github.com/kata-containers/kata-containers/tree/main/src/runtime#configuration
+[4]: https://github.com/kata-containers/runtime#configuration
 [5]: https://docs.docker.com/engine/reference/commandline/dockerd
-[6]: ../../docs/install/docker/ubuntu-docker-install.md
-[7]: ../../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
+[6]: ../docs/install/docker/ubuntu-docker-install.md
+[7]: ../docs/Developer-Guide.md#configure-to-use-initrd-or-rootfs-image
 [8]: https://snapcraft.io/kata-containers
-[9]: ../../docs/Developer-Guide.md#run-kata-containers-with-docker
-[10]: ../../docs/Developer-Guide.md#run-kata-containers-with-kubernetes
+[9]: ../docs/Developer-Guide.md#run-kata-containers-with-docker
+[10]: ../docs/Developer-Guide.md#run-kata-containers-with-kubernetes

snap/local/snap-common.sh

@@ -1,114 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2022 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-
-# Description: Idempotent script to be sourced by all parts in a
-#   snapcraft config file.
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# XXX: Bash-specific code. zsh doesn't support this option and that *does*
-# matter if this script is run sourced... since it'll be using zsh! ;)
-[ -n "$BASH_VERSION" ] && set -o errtrace
-
-[ -n "${DEBUG:-}" ] && set -o xtrace
-
-die()
-{
-	echo >&2 "ERROR: $0: $*"
-}
-
-[ -n "${SNAPCRAFT_STAGE:-}" ] ||\
-	die "must be sourced from a snapcraft config file"
-
-snap_yq_version=3.4.1
-
-snap_common_install_yq()
-{
-      export yq="${SNAPCRAFT_STAGE}/bin/yq"
-
-      local yq_pkg
-      yq_pkg="github.com/mikefarah/yq"
-
-      local yq_url
-      yq_url="https://${yq_pkg}/releases/download/${snap_yq_version}/yq_${goos}_${goarch}"
-      curl -o "${yq}" -L "${yq_url}"
-      chmod +x "${yq}"
-}
-
-# Function that should be called for each snap "part" in
-# snapcraft.yaml.
-snap_common_main()
-{
-	# Architecture
-	arch="$(uname -m)"
-
-	case "${arch}" in
-		aarch64)
-			goarch="arm64"
-			qemu_arch="${arch}"
-			;;
-
-		ppc64le)
-			goarch="ppc64le"
-			qemu_arch="ppc64"
-			;;
-
-		s390x)
-			goarch="${arch}"
-			qemu_arch="${arch}"
-			;;
-
-		x86_64)
-			goarch="amd64"
-			qemu_arch="${arch}"
-			;;
-
-		*) die "unsupported architecture: ${arch}" ;;
-	esac
-
-	dpkg_arch=$(dpkg --print-architecture)
-
-	# golang
-	#
-	# We need the O/S name in golang format, but since we don't
-	# know if the godeps part has run, we don't know if golang is
-	# available yet, hence fall back to a standard system command.
-	goos="$(go env GOOS &>/dev/null || true)"
-	[ -z "$goos" ] && goos=$(uname -s|tr '[A-Z]' '[a-z]')
-
-	export GOROOT="${SNAPCRAFT_STAGE}"
-	export GOPATH="${GOROOT}/gopath"
-	export GO111MODULE="auto"
-
-	mkdir -p "${GOPATH}/bin"
-	export PATH="${GOPATH}/bin:${PATH}"
-
-	# Proxy
-	export http_proxy="${http_proxy:-}"
-	export https_proxy="${https_proxy:-}"
-
-	# Binaries
-	mkdir -p "${SNAPCRAFT_STAGE}/bin"
-
-	export PATH="$PATH:${SNAPCRAFT_STAGE}/bin"
-
-	# YAML query tool
-	export yq="${SNAPCRAFT_STAGE}/bin/yq"
-
-	# Kata paths
-	export kata_dir=$(printf "%s/src/github.com/%s/%s" \
-		"${GOPATH}" \
-		"${SNAPCRAFT_PROJECT_NAME}" \
-		"${SNAPCRAFT_PROJECT_NAME}")
-
-	export versions_file="${kata_dir}/versions.yaml"
-
-	[ -n "${yq:-}" ] && [ -x "${yq:-}" ] || snap_common_install_yq
-}
-
-snap_common_main

snap/snapcraft.yaml

@@ -1,5 +1,4 @@
 name: kata-containers
-website: https://github.com/kata-containers/kata-containers
 summary: Build lightweight VMs that seamlessly plug into the containers ecosystem
 description: |
   Kata Containers is an open source project and community working to build a
@@ -19,18 +18,20 @@ parts:
       - git
       - git-extras
     override-pull: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
       version="9999"
+      kata_url="https://github.com/kata-containers/kata-containers"
 
-      if echo "${GITHUB_REF:-}" | grep -q -E "^refs/tags"; then
-        version=$(echo ${GITHUB_REF:-} | cut -d/ -f3)
+      if echo "${GITHUB_REF}" | grep -q -E "^refs/tags"; then
+        version=$(echo ${GITHUB_REF} | cut -d/ -f3)
         git checkout ${version}
       fi
 
       snapcraftctl set-grade "stable"
       snapcraftctl set-version "${version}"
 
+      # setup GOPATH - this repo dir should be there
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
       mkdir -p $(dirname ${kata_dir})
       ln -sf $(realpath "${SNAPCRAFT_STAGE}/..") ${kata_dir}
 
@@ -42,79 +43,33 @@ parts:
     build-packages:
       - curl
     override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
       # put everything in stage
-      cd "${SNAPCRAFT_STAGE}"
+      cd ${SNAPCRAFT_STAGE}
 
-      version="$(${yq} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
+      yq_path="./yq"
+      yq_pkg="github.com/mikefarah/yq"
+      goos="linux"
+      case "$(uname -m)" in
+        aarch64) goarch="arm64";;
+        ppc64le) goarch="ppc64le";;
+        x86_64) goarch="amd64";;
+        s390x) goarch="s390x";;
+        *) echo "unsupported architecture: $(uname -m)"; exit 1;;
+      esac
+
+      yq_version=3.4.1
+      yq_url="https://${yq_pkg}/releases/download/${yq_version}/yq_${goos}_${goarch}"
+      curl -o "${yq_path}" -L "${yq_url}"
+      chmod +x "${yq_path}"
+
+      kata_dir=gopath/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      version="$(${yq_path} r ${kata_dir}/versions.yaml languages.golang.meta.newest-version)"
       tarfile="go${version}.${goos}-${goarch}.tar.gz"
       curl -LO https://golang.org/dl/${tarfile}
       tar -xf ${tarfile} --strip-components=1
 
-  rustdeps:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - curl
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      # put everything in stage
-      cd "${SNAPCRAFT_STAGE}"
-
-      version="$(${yq} r ${kata_dir}/versions.yaml languages.rust.meta.newest-version)"
-      if ! command -v rustup > /dev/null; then
-        curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${version}
-      fi
-
-      export PATH=${PATH}:${HOME}/.cargo/bin
-      rustup toolchain install ${version}
-      rustup default ${version}
-      if [ "${arch}" == "ppc64le" ] || [ "${arch}" == "s390x" ] ; then
-        [ "${arch}" == "ppc64le" ] && arch="powerpc64le"
-        rustup target add ${arch}-unknown-linux-gnu
-      else
-        rustup target add ${arch}-unknown-linux-musl
-        $([ "$(whoami)" != "root" ] && echo sudo) ln -sf /usr/bin/g++ /bin/musl-g++
-      fi
-      rustup component add rustfmt
-
-  docker:
-    after: [metadata]
-    plugin: nil
-    prime:
-      - -*
-    build-packages:
-      - ca-certificates
-      - containerd
-      - curl
-      - gnupg
-      - lsb-release
-      - runc
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
-          sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-      distro_codename=$(lsb_release -cs)
-      echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
-          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-      sudo apt-get -y update
-      sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-
-      echo "Unmasking docker service"
-      sudo -E systemctl unmask docker.service || true
-      sudo -E systemctl unmask docker.socket || true
-      echo "Adding $USER into docker group"
-      sudo -E gpasswd -a $USER docker
-      echo "Starting docker"
-      sudo -E systemctl start docker || true
-
   image:
-    after: [godeps, docker, qemu, kernel]
+    after: [godeps, qemu, kernel]
     plugin: nil
     build-packages:
       - docker.io
@@ -125,67 +80,95 @@ parts:
       - uidmap
       - gnupg2
     override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+      [ "$(uname -m)" = "ppc64le" ] || [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
 
-      [ "${arch}" = "ppc64le" ] || [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y protobuf-compiler
+      yq=${SNAPCRAFT_STAGE}/yq
 
+      # set GOPATH
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+
+      export GOROOT=${SNAPCRAFT_STAGE}
+      export PATH="${GOROOT}/bin:${PATH}"
+      export GO111MODULE="auto"
+
+      http_proxy=${http_proxy:-""}
+      https_proxy=${https_proxy:-""}
       if [ -n "$http_proxy" ]; then
         echo "Setting proxy $http_proxy"
-        sudo -E systemctl set-environment http_proxy="$http_proxy" || true
-        sudo -E systemctl set-environment https_proxy="$https_proxy" || true
+        sudo -E systemctl set-environment http_proxy=$http_proxy || true
+        sudo -E systemctl set-environment https_proxy=$https_proxy || true
       fi
 
       # Copy yq binary. It's used in the container
+      mkdir -p "${GOPATH}/bin/"
       cp -a "${yq}" "${GOPATH}/bin/"
 
-      cd "${kata_dir}/tools/osbuilder"
+      echo "Unmasking docker service"
+      sudo -E systemctl unmask docker.service || true
+      sudo -E systemctl unmask docker.socket || true
+      echo "Adding $USER into docker group"
+      sudo -E gpasswd -a $USER docker
+      echo "Starting docker"
+      sudo -E systemctl start docker || true
+
+      cd ${kata_dir}/tools/osbuilder
 
       # build image
       export AGENT_INIT=yes
       export USE_DOCKER=1
       export DEBUG=1
+      arch="$(uname -m)"
       initrd_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.initrd.architecture.${arch}.name)
       image_distro=$(${yq} r -X ${kata_dir}/versions.yaml assets.image.architecture.${arch}.name)
       case "$arch" in
         x86_64)
           # In some build systems it's impossible to build a rootfs image, try with the initrd image
-          sudo -E PATH=$PATH make image DISTRO="${image_distro}" || sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
+          sudo -E PATH=$PATH make image DISTRO=${image_distro} || sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
         ;;
 
         aarch64|ppc64le|s390x)
-          sudo -E PATH="$PATH" make initrd DISTRO="${initrd_distro}"
+          sudo -E PATH=$PATH make initrd DISTRO=${initrd_distro}
         ;;
 
-        *) die "unsupported architecture: ${arch}" ;;
+        *) echo "unsupported architecture: $(uname -m)"; exit 1;;
       esac
 
       # Install image
-      kata_image_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
-      mkdir -p "${kata_image_dir}"
-      cp kata-containers*.img "${kata_image_dir}"
+      kata_image_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
+      mkdir -p ${kata_image_dir}
+      cp kata-containers*.img ${kata_image_dir}
 
   runtime:
     after: [godeps, image, cloud-hypervisor]
     plugin: nil
     build-attributes: [no-patchelf]
     override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+      # set GOPATH
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      export GOROOT=${SNAPCRAFT_STAGE}
+      export PATH="${GOROOT}/bin:${PATH}"
+      export GO111MODULE="auto"
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
-      cd "${kata_dir}/src/runtime"
+      cd ${kata_dir}/src/runtime
 
-      qemu_cmd="qemu-system-${qemu_arch}"
+      # setup arch
+      arch=$(uname -m)
+      if [ ${arch} = "ppc64le" ]; then
+        arch="ppc64"
+      fi
 
       # build and install runtime
       make \
-        PREFIX="/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr" \
+        PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr \
         SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD="${qemu_cmd}"
-
+        QEMUCMD=qemu-system-$arch
       make install \
         PREFIX=/usr \
-        DESTDIR="${SNAPCRAFT_PART_INSTALL}" \
+        DESTDIR=${SNAPCRAFT_PART_INSTALL} \
         SKIP_GO_VERSION_CHECK=1 \
-        QEMUCMD="${qemu_cmd}"
+        QEMUCMD=qemu-system-$arch
 
       if [ ! -f ${SNAPCRAFT_PART_INSTALL}/../../image/install/usr/share/kata-containers/kata-containers.img ]; then
         sed -i -e "s|^image =.*|initrd = \"/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr/share/kata-containers/kata-containers-initrd.img\"|" \
@@ -202,37 +185,44 @@ parts:
       - bison
       - flex
     override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
+      yq=${SNAPCRAFT_STAGE}/yq
+      export PATH="${PATH}:${SNAPCRAFT_STAGE}"
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+      versions_file="${kata_dir}/versions.yaml"
       kernel_version="$(${yq} r $versions_file assets.kernel.version)"
       #Remove extra 'v'
-      kernel_version="${kernel_version#v}"
+      kernel_version=${kernel_version#v}
 
-      [ "${arch}" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
+      [ "$(uname -m)" = "s390x" ] && sudo apt-get --no-install-recommends install -y libssl-dev
 
-      cd "${kata_dir}/tools/packaging/kernel"
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      export GO111MODULE="auto"
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+
+      cd ${kata_dir}/tools/packaging/kernel
       kernel_dir_prefix="kata-linux-"
 
       # Setup and build kernel
-      ./build-kernel.sh -v "${kernel_version}" -d setup
+      ./build-kernel.sh -v ${kernel_version} -d setup
       cd ${kernel_dir_prefix}*
-      make -j $(nproc ${CI:+--ignore 1}) EXTRAVERSION=".container"
+      make -j $(($(nproc)-1)) EXTRAVERSION=".container"
 
-      kernel_suffix="${kernel_version}.container"
-      kata_kernel_dir="${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers"
-      mkdir -p "${kata_kernel_dir}"
+      kernel_suffix=${kernel_version}.container
+      kata_kernel_dir=${SNAPCRAFT_PART_INSTALL}/usr/share/kata-containers
+      mkdir -p ${kata_kernel_dir}
 
       # Install bz kernel
-      make install INSTALL_PATH="${kata_kernel_dir}" EXTRAVERSION=".container" || true
-      vmlinuz_name="vmlinuz-${kernel_suffix}"
-      ln -sf "${vmlinuz_name}" "${kata_kernel_dir}/vmlinuz.container"
+      make install INSTALL_PATH=${kata_kernel_dir} EXTRAVERSION=".container" || true
+      vmlinuz_name=vmlinuz-${kernel_suffix}
+      ln -sf ${vmlinuz_name} ${kata_kernel_dir}/vmlinuz.container
 
       # Install raw kernel
-      vmlinux_path="vmlinux"
-      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/vmlinux"
-      vmlinux_name="vmlinux-${kernel_suffix}"
-      cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
-      ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
+      vmlinux_path=vmlinux
+      [ "$(uname -m)" = "s390x" ] && vmlinux_path=arch/s390/boot/compressed/vmlinux
+      vmlinux_name=vmlinux-${kernel_suffix}
+      cp ${vmlinux_path} ${kata_kernel_dir}/${vmlinux_name}
+      ln -sf ${vmlinux_name} ${kata_kernel_dir}/vmlinux.container
 
   qemu:
     plugin: make
@@ -259,8 +249,12 @@ parts:
       - libselinux1-dev
       - ninja-build
     override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+      yq=${SNAPCRAFT_STAGE}/yq
+      export GOPATH=${SNAPCRAFT_STAGE}/gopath
+      export GO111MODULE="auto"
+      kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
 
+      versions_file="${kata_dir}/versions.yaml"
       branch="$(${yq} r ${versions_file} assets.hypervisor.qemu.version)"
       url="$(${yq} r ${versions_file} assets.hypervisor.qemu.url)"
       commit=""
@@ -268,11 +262,11 @@ parts:
       patches_version_dir="${kata_dir}/tools/packaging/qemu/patches/tag_patches/${branch}"
 
       # download source
-      qemu_dir="${SNAPCRAFT_STAGE}/qemu"
+      qemu_dir=${SNAPCRAFT_STAGE}/qemu
       rm -rf "${qemu_dir}"
       git clone --depth 1 --branch ${branch} --single-branch ${url} "${qemu_dir}"
-      cd "${qemu_dir}"
-      [ -z "${commit}" ] || git checkout "${commit}"
+      cd ${qemu_dir}
+      [ -z "${commit}" ] || git checkout ${commit}
 
       [ -n "$(ls -A ui/keycodemapdb)" ] || git clone --depth 1 https://github.com/qemu/keycodemapdb ui/keycodemapdb/
       [ -n "$(ls -A capstone)" ] || git clone --depth 1 https://github.com/qemu/capstone capstone
@@ -283,10 +277,10 @@ parts:
       ${kata_dir}/tools/packaging/scripts/apply_patches.sh "${patches_version_dir}"
 
       # Only x86_64 supports libpmem
-      [ "${arch}" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
+      [ "$(uname -m)" = "x86_64" ] && sudo apt-get --no-install-recommends install -y apt-utils ca-certificates libpmem-dev
 
-      configure_hypervisor="${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh"
-      chmod +x "${configure_hypervisor}"
+      configure_hypervisor=${kata_dir}/tools/packaging/scripts/configure-hypervisor.sh
+      chmod +x ${configure_hypervisor}
       # static build. The --prefix, --libdir, --libexecdir, --datadir arguments are
       # based on PREFIX and set by configure-hypervisor.sh
       echo "$(PREFIX=/snap/${SNAPCRAFT_PROJECT_NAME}/current/usr ${configure_hypervisor} -s kata-qemu) \
@@ -296,17 +290,17 @@ parts:
       # Copy QEMU configurations (Kconfigs)
       case "${branch}" in
       "v5.1.0")
-        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* default-configs
+        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* default-configs
         ;;
 
       *)
-        cp -a "${kata_dir}"/tools/packaging/qemu/default-configs/* configs/devices/
+        cp -a ${kata_dir}/tools/packaging/qemu/default-configs/* configs/devices/
         ;;
       esac
 
       # build and install
-      make -j $(nproc ${CI:+--ignore 1})
-      make install DESTDIR="${SNAPCRAFT_PART_INSTALL}"
+      make -j $(($(nproc)-1))
+      make install DESTDIR=${SNAPCRAFT_PART_INSTALL}
     prime:
       - -snap/
       - -usr/bin/qemu-ga
@@ -322,44 +316,26 @@ parts:
       # Hack: move qemu to /
       "snap/kata-containers/current/": "./"
 
-  virtiofsd:
-    plugin: nil
-    after: [godeps, rustdeps, docker]
-    override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
-
-      echo "INFO: Building rust version of virtiofsd"
-
-      cd "${SNAPCRAFT_PROJECT_DIR}"
-      # Clean-up build dir in case it already exists
-      sudo -E NO_TTY=true make virtiofsd-tarball
-
-      sudo install \
-        --owner='root' \
-        --group='root' \
-        --mode=0755 \
-        -D \
-        --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
-        build/virtiofsd/builddir/virtiofsd/virtiofsd
-
   cloud-hypervisor:
     plugin: nil
-    after: [godeps, docker]
+    after: [godeps]
     override-build: |
-      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+      arch=$(uname -m)
+      if [ "{$arch}" == "aarch64" ] || [ "${arch}" == "x64_64" ]; then 
+          sudo apt-get -y update
+          sudo apt-get -y install ca-certificates curl gnupg lsb-release
+          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+          sudo apt-get -y update
+          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+          sudo systemctl start docker.socket
 
-      if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
-          cd "${SNAPCRAFT_PROJECT_DIR}"
+          export GOPATH=${SNAPCRAFT_STAGE}/gopath
+          kata_dir=${GOPATH}/src/github.com/${SNAPCRAFT_PROJECT_NAME}/${SNAPCRAFT_PROJECT_NAME}
+          cd ${kata_dir}
           sudo -E NO_TTY=true make cloud-hypervisor-tarball
-
-          tarfile="${SNAPCRAFT_PROJECT_DIR}/tools/packaging/kata-deploy/local-build/build/kata-static-cloud-hypervisor.tar.xz"
-          tmpdir=$(mktemp -d)
-
-          tar -xvJpf "${tarfile}" -C "${tmpdir}"
-
-          install -D "${tmpdir}/opt/kata/bin/cloud-hypervisor" "${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor"
-
-          rm -rf "${tmpdir}"
+          tar xvJpf build/kata-static-cloud-hypervisor.tar.xz -C /tmp/
+          install -D /tmp/opt/kata/bin/cloud-hypervisor ${SNAPCRAFT_PART_INSTALL}/usr/bin/cloud-hypervisor
       fi
 
 apps:

src/agent/Cargo.toml

@@ -3,28 +3,24 @@ name = "kata-agent"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
-license = "Apache-2.0"
 
 [dependencies]
 oci = { path = "../libs/oci" }
 rustjail = { path = "rustjail" }
-protocols = { path = "../libs/protocols", features = ["async"] }
+protocols = { path = "../libs/protocols" }
 lazy_static = "1.3.0"
-ttrpc = { version = "0.6.0", features = ["async"], default-features = false }
-protobuf = "2.27.0"
+ttrpc = { version = "0.5.0", features = ["async", "protobuf-codec"], default-features = false }
+protobuf = "=2.14.0"
 libc = "0.2.58"
-nix = "0.24.2"
+nix = "0.23.0"
 capctl = "0.2.0"
 serde_json = "1.0.39"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
-regex = "1.5.6"
+regex = "1.5.4"
 serial_test = "0.5.1"
-kata-sys-util = { path = "../libs/kata-sys-util" }
-kata-types = { path = "../libs/kata-types" }
 sysinfo = "0.23.0"
-url = "2.2.2"
 
 # Async helpers
 async-trait = "0.1.42"
@@ -32,7 +28,7 @@ async-recursion = "0.3.2"
 futures = "0.3.17"
 
 # Async runtime
-tokio = { version = "1.21.2", features = ["full"] }
+tokio = { version = "1.14.0", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
@@ -53,7 +49,7 @@ log = "0.4.11"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.10" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 
 # Tracing
 tracing = "0.1.26"
@@ -67,14 +63,8 @@ serde = { version = "1.0.129", features = ["derive"] }
 toml = "0.5.8"
 clap = { version = "3.0.1", features = ["derive"] }
 
-# Image pull/decrypt
-image-rs = { git = "https://github.com/confidential-containers/image-rs", tag = "v0.2.0" }
-# "vendored" feature for openssl is required by musl build
-openssl = { version = "0.10.38", features = ["vendored"] }
-
 [dev-dependencies]
 tempfile = "3.1.0"
-test-utils = { path = "../libs/test-utils" }
 
 [workspace]
 members = [
@@ -86,8 +76,3 @@ lto = true
 
 [features]
 seccomp = ["rustjail/seccomp"]
-standard-oci-runtime = ["rustjail/standard-oci-runtime"]
-
-[[bin]]
-name = "kata-agent"
-path = "src/main.rs"

src/agent/Makefile

@@ -14,6 +14,10 @@ PROJECT_COMPONENT = kata-agent
 
 TARGET = $(PROJECT_COMPONENT)
 
+SOURCES := \
+  $(shell find . 2>&1 | grep -E '.*\.rs$$') \
+  Cargo.toml
+
 VERSION_FILE := ./VERSION
 VERSION := $(shell grep -v ^\# $(VERSION_FILE))
 COMMIT_NO := $(shell git rev-parse HEAD 2>/dev/null || true)
@@ -33,16 +37,8 @@ ifeq ($(SECCOMP),yes)
     override EXTRA_RUSTFEATURES += seccomp
 endif
 
-##VAR STANDARD_OCI_RUNTIME=yes|no define if agent enables standard oci runtime feature
-STANDARD_OCI_RUNTIME := no
-
-# Enable standard oci runtime feature of rust build
-ifeq ($(STANDARD_OCI_RUNTIME),yes)
-    override EXTRA_RUSTFEATURES += standard-oci-runtime
-endif
-
 ifneq ($(EXTRA_RUSTFEATURES),)
-    override EXTRA_RUSTFEATURES := --features "$(EXTRA_RUSTFEATURES)"
+    override EXTRA_RUSTFEATURES := --features $(EXTRA_RUSTFEATURES)
 endif
 
 include ../../utils.mk
@@ -85,14 +81,6 @@ ifeq ($(INIT),no)
     UNIT_FILES += kata-containers.target
 endif
 
-# The following will be reverted, after
-# https://github.com/kata-containers/kata-containers/issues/5582
-# is resolved.
-IMAGE_RS_COMMIT = a1d7ba31201d9d7a575d05c5fed1f2cb2142a842
-ifeq ($(ARCH),s390x)
-    $(shell sed -i -e "s/^\(image-rs.*\)tag\(.*\)/\1rev\2/" -e "s/^\(image-rs.*rev = \"\).*\(\".*\)/\1$(IMAGE_RS_COMMIT)\2/" Cargo.toml)
-endif
-
 # Display name of command and it's version (or a message if not available).
 #
 # Arguments:
@@ -115,17 +103,20 @@ endef
 ##TARGET default: build code
 default: $(TARGET) show-header
 
-$(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
+$(TARGET): $(GENERATED_CODE) logging-crate-tests $(TARGET_PATH)
 
-$(TARGET_PATH): show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
+logging-crate-tests:
+	make -C $(CWD)/../libs/logging
+
+$(TARGET_PATH): $(SOURCES) | show-summary
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
-optimize: show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
+optimize: $(SOURCES) | show-summary show-header
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET install: install agent
 install: install-services
@@ -208,6 +199,7 @@ codecov-html: check_tarpaulin
 
 .PHONY: \
 	help \
+	logging-crate-tests \
 	optimize \
 	show-header \
 	show-summary \

src/agent/rustjail/Cargo.toml

@@ -3,7 +3,6 @@ name = "rustjail"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
-license = "Apache-2.0"
 
 [dependencies]
 serde = "1.0.91"
@@ -12,33 +11,30 @@ serde_derive = "1.0.91"
 oci = { path = "../../libs/oci" }
 protocols = { path ="../../libs/protocols" }
 caps = "0.5.0"
-nix = "0.24.2"
+nix = "0.23.0"
 scopeguard = "1.0.0"
 capctl = "0.2.0"
 lazy_static = "1.3.0"
 libc = "0.2.58"
-protobuf = "2.27.0"
+protobuf = "=2.14.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2.6"
-regex = "1.5.6"
+regex = "1.5.4"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.10" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
-cfg-if = "0.1.0"
 
-tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros", "rt"] }
+tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
-libseccomp = { version = "0.3.0", optional = true }
+libseccomp = { version = "0.1.3", optional = true }
 
 [dev-dependencies]
 serial_test = "0.5.0"
 tempfile = "3.1.0"
-test-utils = { path = "../../libs/test-utils" }
 
 [features]
 seccomp = ["libseccomp"]
-standard-oci-runtime = []

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -174,7 +174,7 @@ impl CgroupManager for Manager {
                 freezer_controller.freeze()?;
             }
             _ => {
-                return Err(anyhow!("Invalid FreezerState"));
+                return Err(anyhow!(nix::Error::EINVAL));
             }
         }
 
@@ -252,28 +252,19 @@ fn set_devices_resources(
 }
 
 fn set_hugepages_resources(
-    cg: &cgroups::Cgroup,
+    _cg: &cgroups::Cgroup,
     hugepage_limits: &[LinuxHugepageLimit],
     res: &mut cgroups::Resources,
 ) {
     info!(sl!(), "cgroup manager set hugepage");
     let mut limits = vec![];
-    let hugetlb_controller = cg.controller_of::<HugeTlbController>();
 
     for l in hugepage_limits.iter() {
-        if hugetlb_controller.is_some() && hugetlb_controller.unwrap().size_supported(&l.page_size)
-        {
-            let hr = HugePageResource {
-                size: l.page_size.clone(),
-                limit: l.limit,
-            };
-            limits.push(hr);
-        } else {
-            warn!(
-                sl!(),
-                "{} page size support cannot be verified, dropping requested limit", l.page_size
-            );
-        }
+        let hr = HugePageResource {
+            size: l.page_size.clone(),
+            limit: l.limit,
+        };
+        limits.push(hr);
     }
     res.hugepages.limits = limits;
 }
@@ -467,11 +458,8 @@ fn linux_device_to_cgroup_device(d: &LinuxDevice) -> Option<DeviceResource> {
 }
 
 fn linux_device_group_to_cgroup_device(d: &LinuxDeviceCgroup) -> Option<DeviceResource> {
-    let dev_type = match &d.r#type {
-        Some(t_s) => match DeviceType::from_char(t_s.chars().next()) {
-            Some(t_c) => t_c,
-            None => return None,
-        },
+    let dev_type = match DeviceType::from_char(d.r#type.chars().next()) {
+        Some(t) => t,
         None => return None,
     };
 
@@ -528,7 +516,7 @@ lazy_static! {
             // all mknod to all char devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -537,7 +525,7 @@ lazy_static! {
             // all mknod to all block devices
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("b".to_string()),
+                r#type: "b".to_string(),
                 major: Some(WILDCARD),
                 minor: Some(WILDCARD),
                 access: "m".to_string(),
@@ -546,7 +534,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/console
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(1),
                 access: "rwm".to_string(),
@@ -555,7 +543,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/pts/<N>
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(136),
                 minor: Some(WILDCARD),
                 access: "rwm".to_string(),
@@ -564,7 +552,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/ptmx
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(5),
                 minor: Some(2),
                 access: "rwm".to_string(),
@@ -573,7 +561,7 @@ lazy_static! {
             // all read/write/mknod to char device /dev/net/tun
             LinuxDeviceCgroup {
                 allow: true,
-                r#type: Some("c".to_string()),
+                r#type: "c".to_string(),
                 major: Some(10),
                 minor: Some(200),
                 access: "rwm".to_string(),
@@ -923,8 +911,9 @@ pub fn get_paths() -> Result<HashMap<String, String>> {
     Ok(m)
 }
 
-pub fn get_mounts(paths: &HashMap<String, String>) -> Result<HashMap<String, String>> {
+pub fn get_mounts() -> Result<HashMap<String, String>> {
     let mut m = HashMap::new();
+    let paths = get_paths()?;
 
     for l in fs::read_to_string(MOUNTS)?.lines() {
         let p: Vec<&str> = l.splitn(2, " - ").collect();
@@ -962,7 +951,7 @@ impl Manager {
         let mut m = HashMap::new();
 
         let paths = get_paths()?;
-        let mounts = get_mounts(&paths)?;
+        let mounts = get_mounts()?;
 
         for key in paths.keys() {
             let mnt = mounts.get(key);

src/agent/rustjail/src/cgroups/notifier.rs

@@ -151,12 +151,12 @@ async fn register_memory_event(
     let eventfd = eventfd(0, EfdFlags::EFD_CLOEXEC)?;
 
     let event_control_path = Path::new(&cg_dir).join("cgroup.event_control");
-
-    let data = if arg.is_empty() {
-        format!("{} {}", eventfd, event_file.as_raw_fd())
+    let data;
+    if arg.is_empty() {
+        data = format!("{} {}", eventfd, event_file.as_raw_fd());
     } else {
-        format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg)
-    };
+        data = format!("{} {} {}", eventfd, event_file.as_raw_fd(), arg);
+    }
 
     fs::write(&event_control_path, data)?;
 

src/agent/rustjail/src/console.rs

@@ -1,77 +0,0 @@
-// SPDX-License-Identifier: Apache-2.0
-//
-// Copyright 2021 Sony Group Corporation
-//
-
-use anyhow::{anyhow, Result};
-use nix::errno::Errno;
-use nix::pty;
-use nix::sys::socket;
-use nix::unistd::{self, dup2};
-use std::io::IoSlice;
-use std::os::unix::io::{AsRawFd, RawFd};
-use std::path::Path;
-
-pub fn setup_console_socket(csocket_path: &str) -> Result<Option<RawFd>> {
-    if csocket_path.is_empty() {
-        return Ok(None);
-    }
-
-    let socket_fd = socket::socket(
-        socket::AddressFamily::Unix,
-        socket::SockType::Stream,
-        socket::SockFlag::empty(),
-        None,
-    )?;
-
-    match socket::connect(socket_fd, &socket::UnixAddr::new(Path::new(csocket_path))?) {
-        Ok(()) => Ok(Some(socket_fd)),
-        Err(errno) => Err(anyhow!("failed to open console fd: {}", errno)),
-    }
-}
-
-pub fn setup_master_console(socket_fd: RawFd) -> Result<()> {
-    let pseudo = pty::openpty(None, None)?;
-
-    let pty_name: &[u8] = b"/dev/ptmx";
-    let iov = [IoSlice::new(pty_name)];
-    let fds = [pseudo.master];
-    let cmsg = socket::ControlMessage::ScmRights(&fds);
-
-    socket::sendmsg::<()>(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
-
-    unistd::setsid()?;
-    let ret = unsafe { libc::ioctl(pseudo.slave, libc::TIOCSCTTY) };
-    Errno::result(ret).map_err(|e| anyhow!(e).context("ioctl TIOCSCTTY"))?;
-
-    dup2(pseudo.slave, std::io::stdin().as_raw_fd())?;
-    dup2(pseudo.slave, std::io::stdout().as_raw_fd())?;
-    dup2(pseudo.slave, std::io::stderr().as_raw_fd())?;
-
-    unistd::close(socket_fd)?;
-
-    Ok(())
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::os::unix::net::UnixListener;
-    use tempfile::{self, tempdir};
-
-    const CONSOLE_SOCKET: &str = "console-socket";
-
-    #[test]
-    fn test_setup_console_socket() {
-        let dir = tempdir()
-            .map_err(|e| anyhow!(e).context("tempdir failed"))
-            .unwrap();
-        let socket_path = dir.path().join(CONSOLE_SOCKET);
-
-        let _listener = UnixListener::bind(&socket_path).unwrap();
-
-        let ret = setup_console_socket(socket_path.to_str().unwrap());
-
-        assert!(ret.is_ok());
-    }
-}

src/agent/rustjail/src/container.rs

@@ -23,8 +23,6 @@ use crate::cgroups::fs::Manager as FsManager;
 #[cfg(test)]
 use crate::cgroups::mock::Manager as FsManager;
 use crate::cgroups::Manager;
-#[cfg(feature = "standard-oci-runtime")]
-use crate::console;
 use crate::log_child;
 use crate::process::Process;
 #[cfg(feature = "seccomp")]
@@ -42,7 +40,7 @@ use nix::pty;
 use nix::sched::{self, CloneFlags};
 use nix::sys::signal::{self, Signal};
 use nix::sys::stat::{self, Mode};
-use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid, User};
+use nix::unistd::{self, fork, ForkResult, Gid, Pid, Uid};
 use std::os::unix::fs::MetadataExt;
 use std::os::unix::io::AsRawFd;
 
@@ -64,7 +62,9 @@ use rlimit::{setrlimit, Resource, Rlim};
 use tokio::io::AsyncBufReadExt;
 use tokio::sync::Mutex;
 
-pub const EXEC_FIFO_FILENAME: &str = "exec.fifo";
+use crate::utils;
+
+const EXEC_FIFO_FILENAME: &str = "exec.fifo";
 
 const INIT: &str = "INIT";
 const NO_PIVOT: &str = "NO_PIVOT";
@@ -74,7 +74,6 @@ const CLOG_FD: &str = "CLOG_FD";
 const FIFO_FD: &str = "FIFO_FD";
 const HOME_ENV_KEY: &str = "HOME";
 const PIDNS_FD: &str = "PIDNS_FD";
-const CONSOLE_SOCKET_FD: &str = "CONSOLE_SOCKET_FD";
 
 #[derive(Debug)]
 pub struct ContainerStatus {
@@ -83,7 +82,7 @@ pub struct ContainerStatus {
 }
 
 impl ContainerStatus {
-    pub fn new() -> Self {
+    fn new() -> Self {
         ContainerStatus {
             pre_status: ContainerState::Created,
             cur_status: ContainerState::Created,
@@ -100,17 +99,6 @@ impl ContainerStatus {
     }
 }
 
-impl Default for ContainerStatus {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-// We might want to change this to thiserror in the future
-const MissingCGroupManager: &str = "failed to get container's cgroup Manager";
-const MissingLinux: &str = "no linux config";
-const InvalidNamespace: &str = "invalid namespace type";
-
 pub type Config = CreateOpts;
 type NamespaceType = String;
 
@@ -118,7 +106,7 @@ lazy_static! {
     // This locker ensures the child exit signal will be received by the right receiver.
     pub static ref WAIT_PID_LOCKER: Arc<Mutex<bool>> = Arc::new(Mutex::new(false));
 
-    pub static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
+    static ref NAMESPACES: HashMap<&'static str, CloneFlags> = {
         let mut m = HashMap::new();
         m.insert("user", CloneFlags::CLONE_NEWUSER);
         m.insert("ipc", CloneFlags::CLONE_NEWIPC);
@@ -131,7 +119,7 @@ lazy_static! {
     };
 
 // type to name hashmap, better to be in NAMESPACES
-    pub static ref TYPETONAME: HashMap<&'static str, &'static str> = {
+    static ref TYPETONAME: HashMap<&'static str, &'static str> = {
         let mut m = HashMap::new();
         m.insert("ipc", "ipc");
         m.insert("user", "user");
@@ -227,7 +215,7 @@ pub trait BaseContainer {
     async fn start(&mut self, p: Process) -> Result<()>;
     async fn run(&mut self, p: Process) -> Result<()>;
     async fn destroy(&mut self) -> Result<()>;
-    async fn exec(&mut self) -> Result<()>;
+    fn exec(&mut self) -> Result<()>;
 }
 
 // LinuxContainer protected by Mutex
@@ -248,8 +236,6 @@ pub struct LinuxContainer {
     pub status: ContainerStatus,
     pub created: SystemTime,
     pub logger: Logger,
-    #[cfg(feature = "standard-oci-runtime")]
-    pub console_socket: PathBuf,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -297,7 +283,7 @@ impl Container for LinuxContainer {
             self.status.transition(ContainerState::Paused);
             return Ok(());
         }
-        Err(anyhow!(MissingCGroupManager))
+        Err(anyhow!("failed to get container's cgroup manager"))
     }
 
     fn resume(&mut self) -> Result<()> {
@@ -315,7 +301,7 @@ impl Container for LinuxContainer {
             self.status.transition(ContainerState::Running);
             return Ok(());
         }
-        Err(anyhow!(MissingCGroupManager))
+        Err(anyhow!("failed to get container's cgroup manager"))
     }
 }
 
@@ -373,6 +359,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
             )));
         }
     }
+
     log_child!(cfd_log, "child process start run");
     let buf = read_sync(crfd)?;
     let spec_str = std::str::from_utf8(&buf)?;
@@ -392,9 +379,6 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     let cm: FsManager = serde_json::from_str(cm_str)?;
 
-    #[cfg(feature = "standard-oci-runtime")]
-    let csocket_fd = console::setup_console_socket(&std::env::var(CONSOLE_SOCKET_FD)?)?;
-
     let p = if spec.process.is_some() {
         spec.process.as_ref().unwrap()
     } else {
@@ -402,7 +386,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     };
 
     if spec.linux.is_none() {
-        return Err(anyhow!(MissingLinux));
+        return Err(anyhow!("no linux config"));
     }
     let linux = spec.linux.as_ref().unwrap();
 
@@ -416,7 +400,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     for ns in &nses {
         let s = NAMESPACES.get(&ns.r#type.as_str());
         if s.is_none() {
-            return Err(anyhow!(InvalidNamespace));
+            return Err(anyhow!("invalid ns type"));
         }
         let s = s.unwrap();
 
@@ -592,20 +576,14 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     // only change stdio devices owner when user
     // isn't root.
-    if !uid.is_root() {
-        set_stdio_permissions(uid)?;
+    if guser.uid != 0 {
+        set_stdio_permissions(guser.uid)?;
     }
 
     setid(uid, gid)?;
 
     if !guser.additional_gids.is_empty() {
-        let gids: Vec<Gid> = guser
-            .additional_gids
-            .iter()
-            .map(|gid| Gid::from_raw(*gid))
-            .collect();
-
-        unistd::setgroups(&gids).map_err(|e| {
+        setgroups(guser.additional_gids.as_slice()).map_err(|e| {
             let _ = write_sync(
                 cwfd,
                 SYNC_FAILED,
@@ -645,6 +623,11 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         capabilities::drop_privileges(cfd_log, c)?;
     }
 
+    if init {
+        // notify parent to run poststart hooks
+        write_sync(cwfd, SYNC_SUCCESS, "")?;
+    }
+
     let args = oci_process.args.to_vec();
     let env = oci_process.env.to_vec();
 
@@ -666,17 +649,12 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
         }
     }
 
+    // set the "HOME" env getting from "/etc/passwd", if
+    // there's no uid entry in /etc/passwd, set "/" as the
+    // home env.
     if env::var_os(HOME_ENV_KEY).is_none() {
-        // try to set "HOME" env by uid
-        if let Ok(Some(user)) = User::from_uid(Uid::from_raw(guser.uid)) {
-            if let Ok(user_home_dir) = user.dir.into_os_string().into_string() {
-                env::set_var(HOME_ENV_KEY, user_home_dir);
-            }
-        }
-        // set default home dir as "/" if "HOME" env is still empty
-        if env::var_os(HOME_ENV_KEY).is_none() {
-            env::set_var(HOME_ENV_KEY, String::from("/"));
-        }
+        let home_dir = utils::home_dir(guser.uid).unwrap_or_else(|_| String::from("/"));
+        env::set_var(HOME_ENV_KEY, home_dir);
     }
 
     let exec_file = Path::new(&args[0]);
@@ -692,19 +670,10 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let _ = unistd::close(crfd);
     let _ = unistd::close(cwfd);
 
+    unistd::setsid().context("create a new session")?;
     if oci_process.terminal {
-        cfg_if::cfg_if! {
-            if #[cfg(feature = "standard-oci-runtime")] {
-                if let Some(csocket_fd) = csocket_fd {
-                    console::setup_master_console(csocket_fd)?;
-                } else {
-                    return Err(anyhow!("failed to get console master socket fd"));
-                }
-            }
-            else {
-                unistd::setsid().context("create a new session")?;
-                unsafe { libc::ioctl(0, libc::TIOCSCTTY) };
-            }
+        unsafe {
+            libc::ioctl(0, libc::TIOCSCTTY);
         }
     }
 
@@ -736,7 +705,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 // within the container to the specified user.
 // The ownership needs to match because it is created outside of
 // the container and needs to be localized.
-fn set_stdio_permissions(uid: Uid) -> Result<()> {
+fn set_stdio_permissions(uid: libc::uid_t) -> Result<()> {
     let meta = fs::metadata("/dev/null")?;
     let fds = [
         std::io::stdin().as_raw_fd(),
@@ -751,13 +720,19 @@ fn set_stdio_permissions(uid: Uid) -> Result<()> {
             continue;
         }
 
+        // According to the POSIX specification, -1 is used to indicate that owner and group
+        // are not to be changed.  Since uid_t and gid_t are unsigned types, we have to wrap
+        // around to get -1.
+        let gid = 0u32.wrapping_sub(1);
+
         // We only change the uid owner (as it is possible for the mount to
         // prefer a different gid, and there's no reason for us to change it).
         // The reason why we don't just leave the default uid=X mount setup is
         // that users expect to be able to actually use their console. Without
         // this code, you couldn't effectively run as a non-root user inside a
         // container and also have a console set up.
-        unistd::fchown(*fd, Some(uid), None).with_context(|| "set stdio permissions failed")?;
+        let res = unsafe { libc::fchown(*fd, uid, gid) };
+        Errno::result(res).map_err(|e| anyhow!(e).context("set stdio permissions failed"))?;
     }
 
     Ok(())
@@ -953,14 +928,6 @@ impl BaseContainer for LinuxContainer {
 
         let exec_path = std::env::current_exe()?;
         let mut child = std::process::Command::new(exec_path);
-
-        #[allow(unused_mut)]
-        let mut console_name = PathBuf::from("");
-        #[cfg(feature = "standard-oci-runtime")]
-        if !self.console_socket.as_os_str().is_empty() {
-            console_name = self.console_socket.clone();
-        }
-
         let mut child = child
             .arg("init")
             .stdin(child_stdin)
@@ -970,8 +937,7 @@ impl BaseContainer for LinuxContainer {
             .env(NO_PIVOT, format!("{}", self.config.no_pivot_root))
             .env(CRFD_FD, format!("{}", crfd))
             .env(CWFD_FD, format!("{}", cwfd))
-            .env(CLOG_FD, format!("{}", cfd_log))
-            .env(CONSOLE_SOCKET_FD, console_name);
+            .env(CLOG_FD, format!("{}", cfd_log));
 
         if p.init {
             child = child.env(FIFO_FD, format!("{}", fifofd));
@@ -1054,7 +1020,7 @@ impl BaseContainer for LinuxContainer {
         self.start(p).await?;
 
         if init {
-            self.exec().await?;
+            self.exec()?;
             self.status.transition(ContainerState::Running);
         }
 
@@ -1097,22 +1063,12 @@ impl BaseContainer for LinuxContainer {
         fs::remove_dir_all(&self.root)?;
 
         if let Some(cgm) = self.cgroup_manager.as_mut() {
-            // Kill all of the processes created in this container to prevent
-            // the leak of some daemon process when this container shared pidns
-            // with the sandbox.
-            let pids = cgm.get_pids().context("get cgroup pids")?;
-            for i in pids {
-                if let Err(e) = signal::kill(Pid::from_raw(i), Signal::SIGKILL) {
-                    warn!(self.logger, "kill the process {} error: {:?}", i, e);
-                }
-            }
-
             cgm.destroy().context("destroy cgroups")?;
         }
         Ok(())
     }
 
-    async fn exec(&mut self) -> Result<()> {
+    fn exec(&mut self) -> Result<()> {
         let fifo = format!("{}/{}", &self.root, EXEC_FIFO_FILENAME);
         let fd = fcntl::open(fifo.as_str(), OFlag::O_WRONLY, Mode::from_bits_truncate(0))?;
         let data: &[u8] = &[0];
@@ -1124,26 +1080,6 @@ impl BaseContainer for LinuxContainer {
             .as_secs();
 
         self.status.transition(ContainerState::Running);
-
-        let spec = self
-            .config
-            .spec
-            .as_ref()
-            .ok_or_else(|| anyhow!("OCI spec was not found"))?;
-        let st = self.oci_state()?;
-
-        // run poststart hook
-        if spec.hooks.is_some() {
-            info!(self.logger, "poststart hook");
-            let hooks = spec
-                .hooks
-                .as_ref()
-                .ok_or_else(|| anyhow!("OCI hooks were not found"))?;
-            for h in hooks.poststart.iter() {
-                execute_hook(&self.logger, h, &st).await?;
-            }
-        }
-
         unistd::close(fd)?;
 
         Ok(())
@@ -1186,7 +1122,7 @@ fn do_exec(args: &[String]) -> ! {
     unreachable!()
 }
 
-pub fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> Result<()> {
+fn update_namespaces(logger: &Logger, spec: &mut Spec, init_pid: RawFd) -> Result<()> {
     info!(logger, "updating namespaces");
     let linux = spec
         .linux
@@ -1365,6 +1301,20 @@ async fn join_namespaces(
         // notify child run prestart hooks completed
         info!(logger, "notify child run prestart hook completed!");
         write_async(pipe_w, SYNC_SUCCESS, "").await?;
+
+        info!(logger, "notify child parent ready to run poststart hook!");
+        // wait to run poststart hook
+        read_async(pipe_r).await?;
+        info!(logger, "get ready to run poststart hook!");
+
+        // run poststart hook
+        if spec.hooks.is_some() {
+            info!(logger, "poststart hook");
+            let hooks = spec.hooks.as_ref().unwrap();
+            for h in hooks.poststart.iter() {
+                execute_hook(&logger, h, st).await?;
+            }
+        }
     }
 
     info!(logger, "wait for child process ready to run exec");
@@ -1442,10 +1392,18 @@ impl LinuxContainer {
             Some(unistd::getuid()),
             Some(unistd::getgid()),
         )
-        .context(format!("Cannot change owner of container {} root", id))?;
+        .context(format!("cannot change onwer of container {} root", id))?;
+
+        if config.spec.is_none() {
+            return Err(anyhow!(nix::Error::EINVAL));
+        }
 
         let spec = config.spec.as_ref().unwrap();
 
+        if spec.linux.is_none() {
+            return Err(anyhow!(nix::Error::EINVAL));
+        }
+
         let linux = spec.linux.as_ref().unwrap();
 
         let cpath = if linux.cgroups_path.is_empty() {
@@ -1454,12 +1412,7 @@ impl LinuxContainer {
             linux.cgroups_path.clone()
         };
 
-        let cgroup_manager = FsManager::new(cpath.as_str()).map_err(|e| {
-            anyhow!(format!(
-                "fail to create cgroup manager with path {}: {:}",
-                cpath, e
-            ))
-        })?;
+        let cgroup_manager = FsManager::new(cpath.as_str())?;
         info!(logger, "new cgroup_manager {:?}", &cgroup_manager);
 
         Ok(LinuxContainer {
@@ -1478,16 +1431,14 @@ impl LinuxContainer {
                 .unwrap()
                 .as_secs(),
             logger: logger.new(o!("module" => "rustjail", "subsystem" => "container", "cid" => id)),
-            #[cfg(feature = "standard-oci-runtime")]
-            console_socket: Path::new("").to_path_buf(),
         })
     }
+}
 
-    #[cfg(feature = "standard-oci-runtime")]
-    pub fn set_console_socket(&mut self, console_socket: &Path) -> Result<()> {
-        self.console_socket = console_socket.to_path_buf();
-        Ok(())
-    }
+fn setgroups(grps: &[libc::gid_t]) -> Result<()> {
+    let ret = unsafe { libc::setgroups(grps.len(), grps.as_ptr() as *const libc::gid_t) };
+    Errno::result(ret).map(drop)?;
+    Ok(())
 }
 
 use std::fs::OpenOptions;
@@ -1521,13 +1472,13 @@ use std::process::Stdio;
 use std::time::Duration;
 use tokio::io::{AsyncReadExt, AsyncWriteExt};
 
-pub async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
+async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     let logger = logger.new(o!("action" => "execute-hook"));
 
     let binary = PathBuf::from(h.path.as_str());
     let path = binary.canonicalize()?;
     if !path.exists() {
-        return Err(anyhow!("Path {:?} does not exist", path));
+        return Err(anyhow!(nix::Error::EINVAL));
     }
 
     let mut args = h.args.clone();
@@ -1658,12 +1609,11 @@ fn valid_env(e: &str) -> Option<(&str, &str)> {
 mod tests {
     use super::*;
     use crate::process::Process;
-    use nix::unistd::Uid;
+    use crate::skip_if_not_root;
     use std::fs;
     use std::os::unix::fs::MetadataExt;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
-    use test_utils::skip_if_not_root;
     use tokio::process::Command;
 
     macro_rules! sl {
@@ -1805,7 +1755,7 @@ mod tests {
         let old_uid = meta.uid();
 
         let uid = 1000;
-        set_stdio_permissions(Uid::from_raw(uid)).unwrap();
+        set_stdio_permissions(uid).unwrap();
 
         let meta = fs::metadata("/dev/stdin").unwrap();
         assert_eq!(meta.uid(), uid);
@@ -1817,7 +1767,7 @@ mod tests {
         assert_eq!(meta.uid(), uid);
 
         // restore the uid
-        set_stdio_permissions(Uid::from_raw(old_uid)).unwrap();
+        set_stdio_permissions(old_uid).unwrap();
     }
 
     #[test]
@@ -2098,10 +2048,9 @@ mod tests {
         assert!(ret.is_ok(), "Expecting Ok, Got {:?}", ret);
     }
 
-    #[tokio::test]
-    async fn test_linuxcontainer_exec() {
-        let (c, _dir) = new_linux_container();
-        let ret = c.unwrap().exec().await;
+    #[test]
+    fn test_linuxcontainer_exec() {
+        let ret = new_linux_container_and_then(|mut c: LinuxContainer| c.exec());
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
 

src/agent/rustjail/src/lib.rs

@@ -30,8 +30,6 @@ extern crate regex;
 
 pub mod capabilities;
 pub mod cgroups;
-#[cfg(feature = "standard-oci-runtime")]
-pub mod console;
 pub mod container;
 pub mod mount;
 pub mod pipestream;
@@ -41,6 +39,7 @@ pub mod seccomp;
 pub mod specconv;
 pub mod sync;
 pub mod sync_with_async;
+pub mod utils;
 pub mod validator;
 
 use std::collections::HashMap;
@@ -236,12 +235,6 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
     let devices = {
         let mut d = Vec::new();
         for dev in res.Devices.iter() {
-            let dev_type = if dev.Type.is_empty() {
-                None
-            } else {
-                Some(dev.Type.clone())
-            };
-
             let major = if dev.Major == -1 {
                 None
             } else {
@@ -255,7 +248,7 @@ pub fn resources_grpc_to_oci(res: &grpc::LinuxResources) -> oci::LinuxResources
             };
             d.push(oci::LinuxDeviceCgroup {
                 allow: dev.Allow,
-                r#type: dev_type,
+                r#type: dev.Type.clone(),
                 major,
                 minor,
                 access: dev.Access.clone(),
@@ -358,12 +351,13 @@ fn seccomp_grpc_to_oci(sec: &grpc::LinuxSeccomp) -> oci::LinuxSeccomp {
 
         for sys in sec.Syscalls.iter() {
             let mut args = Vec::new();
+            let errno_ret: u32;
 
-            let errno_ret: u32 = if sys.has_errnoret() {
-                sys.get_errnoret()
+            if sys.has_errnoret() {
+                errno_ret = sys.get_errnoret();
             } else {
-                libc::EPERM as u32
-            };
+                errno_ret = libc::EPERM as u32;
+            }
 
             for arg in sys.Args.iter() {
                 args.push(oci::LinuxSeccompArg {
@@ -519,596 +513,13 @@ pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
 
 #[cfg(test)]
 mod tests {
-    use super::*;
-
-    // Parameters:
-    //
-    // 1: expected Result
-    // 2: actual Result
-    // 3: string used to identify the test on error
     #[macro_export]
-    macro_rules! assert_result {
-        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
-            if $expected_result.is_ok() {
-                let expected_value = $expected_result.as_ref().unwrap();
-                let actual_value = $actual_result.unwrap();
-                assert!(*expected_value == actual_value, "{}", $msg);
-            } else {
-                assert!($actual_result.is_err(), "{}", $msg);
-
-                let expected_error = $expected_result.as_ref().unwrap_err();
-                let expected_error_msg = format!("{:?}", expected_error);
-
-                let actual_error_msg = format!("{:?}", $actual_result.unwrap_err());
-
-                assert!(expected_error_msg == actual_error_msg, "{}", $msg);
+    macro_rules! skip_if_not_root {
+        () => {
+            if !nix::unistd::Uid::effective().is_root() {
+                println!("INFO: skipping {} which needs root", module_path!());
+                return;
             }
         };
     }
-
-    #[test]
-    fn test_process_grpc_to_oci() {
-        #[derive(Debug)]
-        struct TestData {
-            grpcproc: grpc::Process,
-            result: oci::Process,
-        }
-
-        let tests = &[
-            TestData {
-                // All fields specified
-                grpcproc: grpc::Process {
-                    Terminal: true,
-                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::some(grpc::Box {
-                        Height: 123,
-                        Width: 456,
-                        ..Default::default()
-                    }),
-                    User: protobuf::SingularPtrField::<grpc::User>::some(grpc::User {
-                        UID: 1234,
-                        GID: 5678,
-                        AdditionalGids: Vec::from([910, 1112]),
-                        Username: String::from("username"),
-                        ..Default::default()
-                    }),
-                    Args: protobuf::RepeatedField::from(Vec::from([
-                        String::from("arg1"),
-                        String::from("arg2"),
-                    ])),
-                    Env: protobuf::RepeatedField::from(Vec::from([String::from("env")])),
-                    Cwd: String::from("cwd"),
-                    Capabilities: protobuf::SingularPtrField::some(grpc::LinuxCapabilities {
-                        Bounding: protobuf::RepeatedField::from(Vec::from([String::from("bnd")])),
-                        Effective: protobuf::RepeatedField::from(Vec::from([String::from("eff")])),
-                        Inheritable: protobuf::RepeatedField::from(Vec::from([String::from(
-                            "inher",
-                        )])),
-                        Permitted: protobuf::RepeatedField::from(Vec::from([String::from("perm")])),
-                        Ambient: protobuf::RepeatedField::from(Vec::from([String::from("amb")])),
-                        ..Default::default()
-                    }),
-                    Rlimits: protobuf::RepeatedField::from(Vec::from([
-                        grpc::POSIXRlimit {
-                            Type: String::from("r#type"),
-                            Hard: 123,
-                            Soft: 456,
-                            ..Default::default()
-                        },
-                        grpc::POSIXRlimit {
-                            Type: String::from("r#type2"),
-                            Hard: 789,
-                            Soft: 1011,
-                            ..Default::default()
-                        },
-                    ])),
-                    NoNewPrivileges: true,
-                    ApparmorProfile: String::from("apparmor profile"),
-                    OOMScoreAdj: 123456,
-                    SelinuxLabel: String::from("Selinux Label"),
-                    ..Default::default()
-                },
-                result: oci::Process {
-                    terminal: true,
-                    console_size: Some(oci::Box {
-                        height: 123,
-                        width: 456,
-                    }),
-                    user: oci::User {
-                        uid: 1234,
-                        gid: 5678,
-                        additional_gids: Vec::from([910, 1112]),
-                        username: String::from("username"),
-                    },
-                    args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                    env: Vec::from([String::from("env")]),
-                    cwd: String::from("cwd"),
-                    capabilities: Some(oci::LinuxCapabilities {
-                        bounding: Vec::from([String::from("bnd")]),
-                        effective: Vec::from([String::from("eff")]),
-                        inheritable: Vec::from([String::from("inher")]),
-                        permitted: Vec::from([String::from("perm")]),
-                        ambient: Vec::from([String::from("amb")]),
-                    }),
-                    rlimits: Vec::from([
-                        oci::PosixRlimit {
-                            r#type: String::from("r#type"),
-                            hard: 123,
-                            soft: 456,
-                        },
-                        oci::PosixRlimit {
-                            r#type: String::from("r#type2"),
-                            hard: 789,
-                            soft: 1011,
-                        },
-                    ]),
-                    no_new_privileges: true,
-                    apparmor_profile: String::from("apparmor profile"),
-                    oom_score_adj: Some(123456),
-                    selinux_label: String::from("Selinux Label"),
-                },
-            },
-            TestData {
-                // None ConsoleSize
-                grpcproc: grpc::Process {
-                    ConsoleSize: protobuf::SingularPtrField::<grpc::Box>::none(),
-                    OOMScoreAdj: 0,
-                    ..Default::default()
-                },
-                result: oci::Process {
-                    console_size: None,
-                    oom_score_adj: Some(0),
-                    ..Default::default()
-                },
-            },
-            TestData {
-                // None User
-                grpcproc: grpc::Process {
-                    User: protobuf::SingularPtrField::<grpc::User>::none(),
-                    OOMScoreAdj: 0,
-                    ..Default::default()
-                },
-                result: oci::Process {
-                    user: oci::User {
-                        uid: 0,
-                        gid: 0,
-                        additional_gids: vec![],
-                        username: String::from(""),
-                    },
-                    oom_score_adj: Some(0),
-                    ..Default::default()
-                },
-            },
-            TestData {
-                // None Capabilities
-                grpcproc: grpc::Process {
-                    Capabilities: protobuf::SingularPtrField::none(),
-                    OOMScoreAdj: 0,
-                    ..Default::default()
-                },
-                result: oci::Process {
-                    capabilities: None,
-                    oom_score_adj: Some(0),
-                    ..Default::default()
-                },
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = process_grpc_to_oci(&d.grpcproc);
-
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            assert_eq!(d.result, result, "{}", msg);
-        }
-    }
-
-    #[test]
-    fn test_root_grpc_to_oci() {
-        #[derive(Debug)]
-        struct TestData {
-            grpcroot: grpc::Root,
-            result: oci::Root,
-        }
-
-        let tests = &[
-            TestData {
-                // Default fields
-                grpcroot: grpc::Root {
-                    ..Default::default()
-                },
-                result: oci::Root {
-                    ..Default::default()
-                },
-            },
-            TestData {
-                // Specified fields, readonly false
-                grpcroot: grpc::Root {
-                    Path: String::from("path"),
-                    Readonly: false,
-                    ..Default::default()
-                },
-                result: oci::Root {
-                    path: String::from("path"),
-                    readonly: false,
-                    ..Default::default()
-                },
-            },
-            TestData {
-                // Specified fields, readonly true
-                grpcroot: grpc::Root {
-                    Path: String::from("path"),
-                    Readonly: true,
-                    ..Default::default()
-                },
-                result: oci::Root {
-                    path: String::from("path"),
-                    readonly: true,
-                    ..Default::default()
-                },
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = root_grpc_to_oci(&d.grpcroot);
-
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            assert_eq!(d.result, result, "{}", msg);
-        }
-    }
-
-    #[test]
-    fn test_hooks_grpc_to_oci() {
-        #[derive(Debug)]
-        struct TestData {
-            grpchooks: grpc::Hooks,
-            result: oci::Hooks,
-        }
-
-        let tests = &[
-            TestData {
-                // Default fields
-                grpchooks: grpc::Hooks {
-                    ..Default::default()
-                },
-                result: oci::Hooks {
-                    ..Default::default()
-                },
-            },
-            TestData {
-                // All specified
-                grpchooks: grpc::Hooks {
-                    Prestart: protobuf::RepeatedField::from(Vec::from([
-                        grpc::Hook {
-                            Path: String::from("prestartpath"),
-                            Args: protobuf::RepeatedField::from(Vec::from([
-                                String::from("arg1"),
-                                String::from("arg2"),
-                            ])),
-                            Env: protobuf::RepeatedField::from(Vec::from([
-                                String::from("env1"),
-                                String::from("env2"),
-                            ])),
-                            Timeout: 10,
-                            ..Default::default()
-                        },
-                        grpc::Hook {
-                            Path: String::from("prestartpath2"),
-                            Args: protobuf::RepeatedField::from(Vec::from([
-                                String::from("arg3"),
-                                String::from("arg4"),
-                            ])),
-                            Env: protobuf::RepeatedField::from(Vec::from([
-                                String::from("env3"),
-                                String::from("env4"),
-                            ])),
-                            Timeout: 25,
-                            ..Default::default()
-                        },
-                    ])),
-                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
-                        Path: String::from("poststartpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
-                        Timeout: 10,
-                        ..Default::default()
-                    }])),
-                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
-                        Path: String::from("poststoppath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
-                        Timeout: 10,
-                        ..Default::default()
-                    }])),
-                    ..Default::default()
-                },
-                result: oci::Hooks {
-                    prestart: Vec::from([
-                        oci::Hook {
-                            path: String::from("prestartpath"),
-                            args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                            env: Vec::from([String::from("env1"), String::from("env2")]),
-                            timeout: Some(10),
-                        },
-                        oci::Hook {
-                            path: String::from("prestartpath2"),
-                            args: Vec::from([String::from("arg3"), String::from("arg4")]),
-                            env: Vec::from([String::from("env3"), String::from("env4")]),
-                            timeout: Some(25),
-                        },
-                    ]),
-                    poststart: Vec::from([oci::Hook {
-                        path: String::from("poststartpath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                    poststop: Vec::from([oci::Hook {
-                        path: String::from("poststoppath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                },
-            },
-            TestData {
-                // Prestart empty
-                grpchooks: grpc::Hooks {
-                    Prestart: protobuf::RepeatedField::from(Vec::from([])),
-                    Poststart: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
-                        Path: String::from("poststartpath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
-                        Timeout: 10,
-                        ..Default::default()
-                    }])),
-                    Poststop: protobuf::RepeatedField::from(Vec::from([grpc::Hook {
-                        Path: String::from("poststoppath"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
-                        Timeout: 10,
-                        ..Default::default()
-                    }])),
-                    ..Default::default()
-                },
-                result: oci::Hooks {
-                    prestart: Vec::from([]),
-                    poststart: Vec::from([oci::Hook {
-                        path: String::from("poststartpath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                    poststop: Vec::from([oci::Hook {
-                        path: String::from("poststoppath"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    }]),
-                },
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = hooks_grpc_to_oci(&d.grpchooks);
-
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            assert_eq!(d.result, result, "{}", msg);
-        }
-    }
-
-    #[test]
-    fn test_mount_grpc_to_oci() {
-        #[derive(Debug)]
-        struct TestData {
-            grpcmount: grpc::Mount,
-            result: oci::Mount,
-        }
-
-        let tests = &[
-            TestData {
-                // Default fields
-                grpcmount: grpc::Mount {
-                    ..Default::default()
-                },
-                result: oci::Mount {
-                    ..Default::default()
-                },
-            },
-            TestData {
-                grpcmount: grpc::Mount {
-                    destination: String::from("destination"),
-                    source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::from([
-                        String::from("option1"),
-                        String::from("option2"),
-                    ])),
-                    ..Default::default()
-                },
-                result: oci::Mount {
-                    destination: String::from("destination"),
-                    source: String::from("source"),
-                    r#type: String::from("fieldtype"),
-                    options: Vec::from([String::from("option1"), String::from("option2")]),
-                },
-            },
-            TestData {
-                grpcmount: grpc::Mount {
-                    destination: String::from("destination"),
-                    source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::new()),
-                    ..Default::default()
-                },
-                result: oci::Mount {
-                    destination: String::from("destination"),
-                    source: String::from("source"),
-                    r#type: String::from("fieldtype"),
-                    options: Vec::new(),
-                },
-            },
-            TestData {
-                grpcmount: grpc::Mount {
-                    destination: String::new(),
-                    source: String::from("source"),
-                    field_type: String::from("fieldtype"),
-                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
-                    ..Default::default()
-                },
-                result: oci::Mount {
-                    destination: String::new(),
-                    source: String::from("source"),
-                    r#type: String::from("fieldtype"),
-                    options: Vec::from([String::from("option1")]),
-                },
-            },
-            TestData {
-                grpcmount: grpc::Mount {
-                    destination: String::from("destination"),
-                    source: String::from("source"),
-                    field_type: String::new(),
-                    options: protobuf::RepeatedField::from(Vec::from([String::from("option1")])),
-                    ..Default::default()
-                },
-                result: oci::Mount {
-                    destination: String::from("destination"),
-                    source: String::from("source"),
-                    r#type: String::new(),
-                    options: Vec::from([String::from("option1")]),
-                },
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = mount_grpc_to_oci(&d.grpcmount);
-
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            assert_eq!(d.result, result, "{}", msg);
-        }
-    }
-
-    #[test]
-    fn test_hook_grpc_to_oci<'a>() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            grpchook: &'a [grpc::Hook],
-            result: Vec<oci::Hook>,
-        }
-
-        let tests = &[
-            TestData {
-                // Default fields
-                grpchook: &[
-                    grpc::Hook {
-                        Timeout: 0,
-                        ..Default::default()
-                    },
-                    grpc::Hook {
-                        Timeout: 0,
-                        ..Default::default()
-                    },
-                ],
-                result: vec![
-                    oci::Hook {
-                        timeout: Some(0),
-                        ..Default::default()
-                    },
-                    oci::Hook {
-                        timeout: Some(0),
-                        ..Default::default()
-                    },
-                ],
-            },
-            TestData {
-                // Specified fields
-                grpchook: &[
-                    grpc::Hook {
-                        Path: String::from("path"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg1"),
-                            String::from("arg2"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env1"),
-                            String::from("env2"),
-                        ])),
-                        Timeout: 10,
-                        ..Default::default()
-                    },
-                    grpc::Hook {
-                        Path: String::from("path2"),
-                        Args: protobuf::RepeatedField::from(Vec::from([
-                            String::from("arg3"),
-                            String::from("arg4"),
-                        ])),
-                        Env: protobuf::RepeatedField::from(Vec::from([
-                            String::from("env3"),
-                            String::from("env4"),
-                        ])),
-                        Timeout: 20,
-                        ..Default::default()
-                    },
-                ],
-                result: vec![
-                    oci::Hook {
-                        path: String::from("path"),
-                        args: Vec::from([String::from("arg1"), String::from("arg2")]),
-                        env: Vec::from([String::from("env1"), String::from("env2")]),
-                        timeout: Some(10),
-                    },
-                    oci::Hook {
-                        path: String::from("path2"),
-                        args: Vec::from([String::from("arg3"), String::from("arg4")]),
-                        env: Vec::from([String::from("env3"), String::from("env4")]),
-                        timeout: Some(20),
-                    },
-                ],
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = hook_grpc_to_oci(d.grpchook);
-
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            assert_eq!(d.result, result, "{}", msg);
-        }
-    }
 }

src/agent/rustjail/src/mount.rs

@@ -32,21 +32,16 @@ use crate::log_child;
 
 // Info reveals information about a particular mounted filesystem. This
 // struct is populated from the content in the /proc/<pid>/mountinfo file.
-#[derive(std::fmt::Debug, PartialEq)]
+#[derive(std::fmt::Debug)]
 pub struct Info {
     mount_point: String,
     optional: String,
     fstype: String,
 }
 
-const MOUNTINFO_FORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
-const MOUNTINFO_PATH: &str = "/proc/self/mountinfo";
+const MOUNTINFOFORMAT: &str = "{d} {d} {d}:{d} {} {} {} {}";
 const PROC_PATH: &str = "/proc";
 
-const ERR_FAILED_PARSE_MOUNTINFO: &str = "failed to parse mountinfo file";
-const ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS: &str =
-    "failed to parse final fields in mountinfo file";
-
 // since libc didn't defined this const for musl, thus redefined it here.
 #[cfg(all(target_os = "linux", target_env = "gnu", not(target_arch = "s390x")))]
 const PROC_SUPER_MAGIC: libc::c_long = 0x00009fa0;
@@ -523,7 +518,7 @@ pub fn pivot_rootfs<P: ?Sized + NixPath + std::fmt::Debug>(path: &P) -> Result<(
 }
 
 fn rootfs_parent_mount_private(path: &str) -> Result<()> {
-    let mount_infos = parse_mount_table(MOUNTINFO_PATH)?;
+    let mount_infos = parse_mount_table()?;
 
     let mut max_len = 0;
     let mut mount_point = String::from("");
@@ -551,8 +546,8 @@ fn rootfs_parent_mount_private(path: &str) -> Result<()> {
 
 // Parse /proc/self/mountinfo because comparing Dev and ino does not work from
 // bind mounts
-fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
-    let file = File::open(mountinfo_path)?;
+fn parse_mount_table() -> Result<Vec<Info>> {
+    let file = File::open("/proc/self/mountinfo")?;
     let reader = BufReader::new(file);
     let mut infos = Vec::new();
 
@@ -574,7 +569,7 @@ fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
 
         let (_id, _parent, _major, _minor, _root, mount_point, _opts, optional) = scan_fmt!(
             &line,
-            MOUNTINFO_FORMAT,
+            MOUNTINFOFORMAT,
             i32,
             i32,
             i32,
@@ -583,17 +578,12 @@ fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
             String,
             String,
             String
-        )
-        .map_err(|_| anyhow!(ERR_FAILED_PARSE_MOUNTINFO))?;
+        )?;
 
         let fields: Vec<&str> = line.split(" - ").collect();
         if fields.len() == 2 {
-            let final_fields: Vec<&str> = fields[1].split_whitespace().collect();
-
-            if final_fields.len() != 3 {
-                return Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS));
-            }
-            let fstype = final_fields[0].to_string();
+            let (fstype, _source, _vfs_opts) =
+                scan_fmt!(fields[1], "{} {} {}", String, String, String)?;
 
             let mut optional_new = String::new();
             if optional != "-" {
@@ -608,7 +598,7 @@ fn parse_mount_table(mountinfo_path: &str) -> Result<Vec<Info>> {
 
             infos.push(info);
         } else {
-            return Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO));
+            return Err(anyhow!("failed to parse mount info file".to_string()));
         }
     }
 
@@ -629,7 +619,7 @@ fn chroot<P: ?Sized + NixPath>(_path: &P) -> Result<(), nix::Error> {
 
 pub fn ms_move_root(rootfs: &str) -> Result<bool> {
     unistd::chdir(rootfs)?;
-    let mount_infos = parse_mount_table(MOUNTINFO_PATH)?;
+    let mount_infos = parse_mount_table()?;
 
     let root_path = Path::new(rootfs);
     let abs_root_buf = root_path.absolutize()?;
@@ -780,31 +770,18 @@ fn mount_from(
             Path::new(&dest).parent().unwrap()
         };
 
-        fs::create_dir_all(&dir).map_err(|e| {
+        let _ = fs::create_dir_all(&dir).map_err(|e| {
             log_child!(
                 cfd_log,
                 "create dir {}: {}",
                 dir.to_str().unwrap(),
                 e.to_string()
-            );
-            e
-        })?;
+            )
+        });
 
         // make sure file exists so we can bind over it
         if !src.is_dir() {
-            let _ = OpenOptions::new()
-                .create(true)
-                .write(true)
-                .open(&dest)
-                .map_err(|e| {
-                    log_child!(
-                        cfd_log,
-                        "open/create dest error. {}: {:?}",
-                        dest.as_str(),
-                        e
-                    );
-                    e
-                })?;
+            let _ = OpenOptions::new().create(true).write(true).open(&dest);
         }
         src.to_str().unwrap().to_string()
     } else {
@@ -817,10 +794,8 @@ fn mount_from(
         }
     };
 
-    let _ = stat::stat(dest.as_str()).map_err(|e| {
-        log_child!(cfd_log, "dest stat error. {}: {:?}", dest.as_str(), e);
-        e
-    })?;
+    let _ = stat::stat(dest.as_str())
+        .map_err(|e| log_child!(cfd_log, "dest stat error. {}: {:?}", dest.as_str(), e));
 
     mount(
         Some(src.as_str()),
@@ -1020,7 +995,9 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec, process: &Process) -> Result<(
 }
 
 fn mask_path(path: &str) -> Result<()> {
-    check_paths(path)?;
+    if !path.starts_with('/') || path.contains("..") {
+        return Err(anyhow!(nix::Error::EINVAL));
+    }
 
     match mount(
         Some("/dev/null"),
@@ -1038,7 +1015,9 @@ fn mask_path(path: &str) -> Result<()> {
 }
 
 fn readonly_path(path: &str) -> Result<()> {
-    check_paths(path)?;
+    if !path.starts_with('/') || path.contains("..") {
+        return Err(anyhow!(nix::Error::EINVAL));
+    }
 
     if let Err(e) = mount(
         Some(&path[1..]),
@@ -1064,28 +1043,16 @@ fn readonly_path(path: &str) -> Result<()> {
     Ok(())
 }
 
-fn check_paths(path: &str) -> Result<()> {
-    if !path.starts_with('/') || path.contains("..") {
-        return Err(anyhow!(
-            "Cannot mount {} (path does not start with '/' or contains '..').",
-            path
-        ));
-    }
-    Ok(())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::assert_result;
+    use crate::skip_if_not_root;
     use std::fs::create_dir;
     use std::fs::create_dir_all;
     use std::fs::remove_dir_all;
-    use std::io;
     use std::os::unix::fs;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
-    use test_utils::skip_if_not_root;
 
     #[test]
     #[serial(chdir)]
@@ -1319,162 +1286,6 @@ mod tests {
         let ret = stat::stat(path);
         assert!(ret.is_ok(), "Should pass. Got: {:?}", ret);
     }
-
-    #[test]
-    fn test_mount_from() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            source: &'a str,
-            destination: &'a str,
-            r#type: &'a str,
-            flags: MsFlags,
-            error_contains: &'a str,
-
-            // if true, a directory will be created at path in source
-            make_source_directory: bool,
-            // if true, a file will be created at path in source
-            make_source_file: bool,
-        }
-
-        impl Default for TestData<'_> {
-            fn default() -> Self {
-                TestData {
-                    source: "tmp",
-                    destination: "dest",
-                    r#type: "tmpfs",
-                    flags: MsFlags::empty(),
-                    error_contains: "",
-                    make_source_directory: true,
-                    make_source_file: false,
-                }
-            }
-        }
-
-        let tests = &[
-            TestData {
-                ..Default::default()
-            },
-            TestData {
-                flags: MsFlags::MS_BIND,
-                ..Default::default()
-            },
-            TestData {
-                r#type: "bind",
-                ..Default::default()
-            },
-            TestData {
-                r#type: "cgroup2",
-                ..Default::default()
-            },
-            TestData {
-                r#type: "bind",
-                make_source_directory: false,
-                error_contains: &format!("{}", std::io::Error::from_raw_os_error(libc::ENOENT)),
-                ..Default::default()
-            },
-            TestData {
-                r#type: "bind",
-                make_source_directory: false,
-                make_source_file: true,
-                ..Default::default()
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-            let tempdir = tempdir().unwrap();
-
-            let (rfd, wfd) = unistd::pipe2(OFlag::O_CLOEXEC).unwrap();
-            defer!({
-                unistd::close(rfd).unwrap();
-                unistd::close(wfd).unwrap();
-            });
-
-            let source_path = tempdir.path().join(d.source).to_str().unwrap().to_string();
-            if d.make_source_directory {
-                std::fs::create_dir_all(&source_path).unwrap();
-            } else if d.make_source_file {
-                std::fs::write(&source_path, []).unwrap();
-            }
-
-            let mount = Mount {
-                source: source_path,
-                destination: d.destination.to_string(),
-                r#type: d.r#type.to_string(),
-                options: vec![],
-            };
-
-            let result = mount_from(
-                wfd,
-                &mount,
-                tempdir.path().to_str().unwrap(),
-                d.flags,
-                "",
-                "",
-            );
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            if d.error_contains.is_empty() {
-                assert!(result.is_ok(), "{}", msg);
-            } else {
-                assert!(result.is_err(), "{}", msg);
-
-                let error_msg = format!("{}", result.unwrap_err());
-                assert!(error_msg.contains(d.error_contains), "{}", msg);
-            }
-        }
-    }
-
-    #[test]
-    fn test_check_paths() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            name: &'a str,
-            path: &'a str,
-            result: Result<()>,
-        }
-
-        let tests = &[
-            TestData {
-                name: "valid path",
-                path: "/foo/bar",
-                result: Ok(()),
-            },
-            TestData {
-                name: "does not starts with /",
-                path: "foo/bar",
-                result: Err(anyhow!(
-                    "Cannot mount foo/bar (path does not start with '/' or contains '..')."
-                )),
-            },
-            TestData {
-                name: "contains ..",
-                path: "../foo/bar",
-                result: Err(anyhow!(
-                    "Cannot mount ../foo/bar (path does not start with '/' or contains '..')."
-                )),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d.name);
-
-            let result = check_paths(d.path);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            if d.result.is_ok() {
-                assert!(result.is_ok());
-                continue;
-            }
-
-            let expected_error = format!("{}", d.result.as_ref().unwrap_err());
-            let actual_error = format!("{}", result.unwrap_err());
-            assert!(actual_error == expected_error, "{}", msg);
-        }
-    }
-
     #[test]
     fn test_check_proc_mount() {
         let mount = oci::Mount {
@@ -1590,121 +1401,6 @@ mod tests {
         }
     }
 
-    #[test]
-    fn test_parse_mount_table() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            mountinfo_data: Option<&'a str>,
-            result: Result<Vec<Info>>,
-        }
-
-        let tests = &[
-            TestData {
-                mountinfo_data: Some(
-                    "22 933 0:20 / /sys rw,nodev shared:2 - sysfs sysfs rw,noexec",
-                ),
-                result: Ok(vec![Info {
-                    mount_point: "/sys".to_string(),
-                    optional: "shared:2".to_string(),
-                    fstype: "sysfs".to_string(),
-                }]),
-            },
-            TestData {
-                mountinfo_data: Some(
-                    r#"22 933 0:20 / /sys rw,nodev - sysfs sysfs rw,noexec
-                       81 13 1:2 / /tmp/dir rw shared:2 - tmpfs tmpfs rw"#,
-                ),
-                result: Ok(vec![
-                    Info {
-                        mount_point: "/sys".to_string(),
-                        optional: "".to_string(),
-                        fstype: "sysfs".to_string(),
-                    },
-                    Info {
-                        mount_point: "/tmp/dir".to_string(),
-                        optional: "shared:2".to_string(),
-                        fstype: "tmpfs".to_string(),
-                    },
-                ]),
-            },
-            TestData {
-                mountinfo_data: Some(
-                    "22 933 0:20 /foo\040-\040bar /sys rw,nodev shared:2 - sysfs sysfs rw,noexec",
-                ),
-                result: Ok(vec![Info {
-                    mount_point: "/sys".to_string(),
-                    optional: "shared:2".to_string(),
-                    fstype: "sysfs".to_string(),
-                }]),
-            },
-            TestData {
-                mountinfo_data: Some(""),
-                result: Ok(vec![]),
-            },
-            TestData {
-                mountinfo_data: Some("invalid line data - sysfs sysfs rw"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: Some("22 96 0:21 / /sys rw,noexec - sysfs"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS)),
-            },
-            TestData {
-                mountinfo_data: Some("22 96 0:21 / /sys rw,noexec - sysfs sysfs rw rw"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO_FINAL_FIELDS)),
-            },
-            TestData {
-                mountinfo_data: Some("22 96 0:21 / /sys rw,noexec shared:2 - x - x"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: Some("-"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: Some("--"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: Some("- -"),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: Some(" - "),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: Some(
-                    r#"22 933 0:20 / /sys rw,nodev - sysfs sysfs rw,noexec
-                       invalid line
-                       81 13 1:2 / /tmp/dir rw shared:2 - tmpfs tmpfs rw"#,
-                ),
-                result: Err(anyhow!(ERR_FAILED_PARSE_MOUNTINFO)),
-            },
-            TestData {
-                mountinfo_data: None,
-                result: Err(anyhow!(io::Error::from_raw_os_error(libc::ENOENT))),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let tempdir = tempdir().unwrap();
-            let mountinfo_path = tempdir.path().join("mountinfo");
-
-            if let Some(mountinfo_data) = d.mountinfo_data {
-                std::fs::write(&mountinfo_path, mountinfo_data).unwrap();
-            }
-
-            let result = parse_mount_table(mountinfo_path.to_str().unwrap());
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
-    }
-
     #[test]
     fn test_dev_rel_path() {
         // Valid device paths

src/agent/rustjail/src/process.rs

@@ -5,7 +5,7 @@
 
 use libc::pid_t;
 use std::fs::File;
-use std::os::unix::io::{AsRawFd, RawFd};
+use std::os::unix::io::RawFd;
 use tokio::sync::mpsc::Sender;
 
 use nix::errno::Errno;
@@ -28,6 +28,7 @@ macro_rules! close_process_stream {
     ($self: ident, $stream:ident, $stream_type: ident) => {
         if $self.$stream.is_some() {
             $self.close_stream(StreamType::$stream_type);
+            let _ = unistd::close($self.$stream.unwrap());
             $self.$stream = None;
         }
     };
@@ -136,25 +137,19 @@ impl Process {
         info!(logger, "before create console socket!");
 
         if !p.tty {
-            if cfg!(feature = "standard-oci-runtime") {
-                p.stdin = Some(std::io::stdin().as_raw_fd());
-                p.stdout = Some(std::io::stdout().as_raw_fd());
-                p.stderr = Some(std::io::stderr().as_raw_fd());
-            } else {
-                info!(logger, "created console socket!");
+            info!(logger, "created console socket!");
 
-                let (stdin, pstdin) = unistd::pipe2(OFlag::O_CLOEXEC)?;
-                p.parent_stdin = Some(pstdin);
-                p.stdin = Some(stdin);
+            let (stdin, pstdin) = unistd::pipe2(OFlag::O_CLOEXEC)?;
+            p.parent_stdin = Some(pstdin);
+            p.stdin = Some(stdin);
 
-                let (pstdout, stdout) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
-                p.parent_stdout = Some(pstdout);
-                p.stdout = Some(stdout);
+            let (pstdout, stdout) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
+            p.parent_stdout = Some(pstdout);
+            p.stdout = Some(stdout);
 
-                let (pstderr, stderr) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
-                p.parent_stderr = Some(pstderr);
-                p.stderr = Some(stderr);
-            }
+            let (pstderr, stderr) = create_extended_pipe(OFlag::O_CLOEXEC, pipe_size)?;
+            p.parent_stderr = Some(pstderr);
+            p.stderr = Some(stderr);
         }
         Ok(p)
     }
@@ -224,7 +219,7 @@ impl Process {
         Some(writer)
     }
 
-    fn close_stream(&mut self, stream_type: StreamType) {
+    pub fn close_stream(&mut self, stream_type: StreamType) {
         let _ = self.readers.remove(&stream_type);
         let _ = self.writers.remove(&stream_type);
     }
@@ -289,11 +284,5 @@ mod tests {
         // group of the calling process.
         process.pid = 0;
         assert!(process.signal(libc::SIGCONT).is_ok());
-
-        if cfg!(feature = "standard-oci-runtime") {
-            assert_eq!(process.stdin.unwrap(), std::io::stdin().as_raw_fd());
-            assert_eq!(process.stdout.unwrap(), std::io::stdout().as_raw_fd());
-            assert_eq!(process.stderr.unwrap(), std::io::stderr().as_raw_fd());
-        }
     }
 }

src/agent/rustjail/src/seccomp.rs

@@ -26,15 +26,12 @@ fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>>
             return Err(anyhow!("seccomp opreator is required"));
         }
 
-        let mut op = ScmpCompareOp::from_str(&arg.op)?;
-        let mut value = arg.value;
-        // For SCMP_CMP_MASKED_EQ, arg.value is the mask and arg.value_two is the value
-        if op == ScmpCompareOp::MaskedEqual(u64::default()) {
-            op = ScmpCompareOp::MaskedEqual(arg.value);
-            value = arg.value_two;
-        }
-
-        let cond = ScmpArgCompare::new(arg.index, op, value);
+        let cond = ScmpArgCompare::new(
+            arg.index,
+            ScmpCompareOp::from_str(&arg.op)?,
+            arg.value,
+            Some(arg.value_two),
+        );
 
         conditions.push(cond);
     }
@@ -47,7 +44,7 @@ pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
 
     for syscall in &scmp.syscalls {
         for name in &syscall.names {
-            if ScmpSyscall::from_name(name).is_err() {
+            if get_syscall_from_name(name, None).is_err() {
                 unknown_syscalls.push(name.to_string());
             }
         }
@@ -63,7 +60,7 @@ pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
 // init_seccomp creates a seccomp filter and loads it for the current process
 // including all the child processes.
 pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
-    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as i32))?;
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as u32))?;
 
     // Create a new filter context
     let mut filter = ScmpFilterContext::new_filter(def_action)?;
@@ -75,7 +72,7 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
     }
 
     // Unset no new privileges bit
-    filter.set_ctl_nnp(false)?;
+    filter.set_no_new_privs_bit(false)?;
 
     // Add a rule for each system call
     for syscall in &scmp.syscalls {
@@ -83,13 +80,13 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
             return Err(anyhow!("syscall name is required"));
         }
 
-        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret as i32))?;
+        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret))?;
         if action == def_action {
             continue;
         }
 
         for name in &syscall.names {
-            let syscall_num = match ScmpSyscall::from_name(name) {
+            let syscall_num = match get_syscall_from_name(name, None) {
                 Ok(num) => num,
                 Err(_) => {
                     // If we cannot resolve the given system call, we assume it is not supported
@@ -99,10 +96,10 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
             };
 
             if syscall.args.is_empty() {
-                filter.add_rule(action, syscall_num)?;
+                filter.add_rule(action, syscall_num, None)?;
             } else {
                 let conditions = get_rule_conditions(&syscall.args)?;
-                filter.add_rule_conditional(action, syscall_num, &conditions)?;
+                filter.add_rule(action, syscall_num, Some(&conditions))?;
             }
         }
     }
@@ -122,10 +119,10 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use crate::skip_if_not_root;
     use libc::{dup3, process_vm_readv, EPERM, O_CLOEXEC};
     use std::io::Error;
     use std::ptr::null;
-    use test_utils::skip_if_not_root;
 
     macro_rules! syscall_assert {
         ($e1: expr, $e2: expr) => {

src/agent/rustjail/src/specconv.rs

@@ -5,7 +5,7 @@
 
 use oci::Spec;
 
-#[derive(Serialize, Deserialize, Debug, Default, Clone)]
+#[derive(Debug)]
 pub struct CreateOpts {
     pub cgroup_name: String,
     pub use_systemd_cgroup: bool,

src/agent/rustjail/src/utils.rs

@@ -0,0 +1,119 @@
+// Copyright (c) 2021 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+use anyhow::{anyhow, Context, Result};
+use libc::gid_t;
+use libc::uid_t;
+use std::fs::File;
+use std::io::{BufRead, BufReader};
+
+const PASSWD_FILE: &str = "/etc/passwd";
+
+// An entry from /etc/passwd
+#[derive(Debug, PartialEq, PartialOrd)]
+pub struct PasswdEntry {
+    // username
+    pub name: String,
+    // user password
+    pub passwd: String,
+    // user id
+    pub uid: uid_t,
+    // group id
+    pub gid: gid_t,
+    // user Information
+    pub gecos: String,
+    // home directory
+    pub dir: String,
+    // User's Shell
+    pub shell: String,
+}
+
+// get an entry for a given `uid` from `/etc/passwd`
+fn get_entry_by_uid(uid: uid_t, path: &str) -> Result<PasswdEntry> {
+    let file = File::open(path).with_context(|| format!("open file {}", path))?;
+    let mut reader = BufReader::new(file);
+
+    let mut line = String::new();
+    loop {
+        line.clear();
+        match reader.read_line(&mut line) {
+            Ok(0) => return Err(anyhow!(format!("file {} is empty", path))),
+            Ok(_) => (),
+            Err(e) => {
+                return Err(anyhow!(format!(
+                    "failed to read file {} with {:?}",
+                    path, e
+                )))
+            }
+        }
+
+        if line.starts_with('#') {
+            continue;
+        }
+
+        let parts: Vec<&str> = line.split(':').map(|part| part.trim()).collect();
+        if parts.len() != 7 {
+            continue;
+        }
+
+        match parts[2].parse() {
+            Err(_e) => continue,
+            Ok(new_uid) => {
+                if uid != new_uid {
+                    continue;
+                }
+
+                let entry = PasswdEntry {
+                    name: parts[0].to_string(),
+                    passwd: parts[1].to_string(),
+                    uid: new_uid,
+                    gid: parts[3].parse().unwrap_or(0),
+                    gecos: parts[4].to_string(),
+                    dir: parts[5].to_string(),
+                    shell: parts[6].to_string(),
+                };
+
+                return Ok(entry);
+            }
+        }
+    }
+}
+
+pub fn home_dir(uid: uid_t) -> Result<String> {
+    get_entry_by_uid(uid, PASSWD_FILE).map(|entry| entry.dir)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::io::Write;
+    use tempfile::Builder;
+
+    #[test]
+    fn test_get_entry_by_uid() {
+        let tmpdir = Builder::new().tempdir().unwrap();
+        let tmpdir_path = tmpdir.path().to_str().unwrap();
+        let temp_passwd = format!("{}/passwd", tmpdir_path);
+
+        let mut tempf = File::create(temp_passwd.as_str()).unwrap();
+        writeln!(tempf, "root:x:0:0:root:/root0:/bin/bash").unwrap();
+        writeln!(tempf, "root:x:1:0:root:/root1:/bin/bash").unwrap();
+        writeln!(tempf, "#root:x:1:0:root:/rootx:/bin/bash").unwrap();
+        writeln!(tempf, "root:x:2:0:root:/root2:/bin/bash").unwrap();
+        writeln!(tempf, "root:x:3:0:root:/root3").unwrap();
+        writeln!(tempf, "root:x:3:0:root:/root3:/bin/bash").unwrap();
+
+        let entry = get_entry_by_uid(0, temp_passwd.as_str()).unwrap();
+        assert_eq!(entry.dir.as_str(), "/root0");
+
+        let entry = get_entry_by_uid(1, temp_passwd.as_str()).unwrap();
+        assert_eq!(entry.dir.as_str(), "/root1");
+
+        let entry = get_entry_by_uid(2, temp_passwd.as_str()).unwrap();
+        assert_eq!(entry.dir.as_str(), "/root2");
+
+        let entry = get_entry_by_uid(3, temp_passwd.as_str()).unwrap();
+        assert_eq!(entry.dir.as_str(), "/root3");
+    }
+}

src/agent/rustjail/src/validator.rs

@@ -4,15 +4,17 @@
 //
 
 use crate::container::Config;
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Context, Error, Result};
 use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
+fn einval() -> Error {
+    anyhow!(nix::Error::EINVAL)
+}
+
 fn get_linux(oci: &Spec) -> Result<&Linux> {
-    oci.linux
-        .as_ref()
-        .ok_or_else(|| anyhow!("Unable to get Linux section from Spec"))
+    oci.linux.as_ref().ok_or_else(einval)
 }
 
 fn contain_namespace(nses: &[LinuxNamespace], key: &str) -> bool {
@@ -29,10 +31,7 @@ fn rootfs(root: &str) -> Result<()> {
     let path = PathBuf::from(root);
     // not absolute path or not exists
     if !path.exists() || !path.is_absolute() {
-        return Err(anyhow!(
-            "Path from {:?} does not exist or is not absolute",
-            root
-        ));
+        return Err(einval());
     }
 
     // symbolic link? ..?
@@ -50,7 +49,7 @@ fn rootfs(root: &str) -> Result<()> {
         if let Some(v) = c.as_os_str().to_str() {
             stack.push(v.to_string());
         } else {
-            return Err(anyhow!("Invalid path component (unable to convert to str)"));
+            return Err(einval());
         }
     }
 
@@ -59,13 +58,10 @@ fn rootfs(root: &str) -> Result<()> {
         cleaned.push(e);
     }
 
-    let canon = path.canonicalize().context("failed to canonicalize path")?;
+    let canon = path.canonicalize().context("canonicalize")?;
     if cleaned != canon {
         // There is symbolic in path
-        return Err(anyhow!(
-            "There may be illegal symbols in the path name. Cleaned ({:?}) and canonicalized ({:?}) paths do not match",
-            cleaned,
-            canon));
+        return Err(einval());
     }
 
     Ok(())
@@ -78,7 +74,7 @@ fn hostname(oci: &Spec) -> Result<()> {
 
     let linux = get_linux(oci)?;
     if !contain_namespace(&linux.namespaces, "uts") {
-        return Err(anyhow!("Linux namespace does not contain uts"));
+        return Err(einval());
     }
 
     Ok(())
@@ -92,7 +88,7 @@ fn security(oci: &Spec) -> Result<()> {
     }
 
     if !contain_namespace(&linux.namespaces, "mount") {
-        return Err(anyhow!("Linux namespace does not contain mount"));
+        return Err(einval());
     }
 
     // don't care about selinux at present
@@ -107,7 +103,7 @@ fn idmapping(maps: &[LinuxIdMapping]) -> Result<()> {
         }
     }
 
-    Err(anyhow!("No idmap has size > 0"))
+    Err(einval())
 }
 
 fn usernamespace(oci: &Spec) -> Result<()> {
@@ -125,7 +121,7 @@ fn usernamespace(oci: &Spec) -> Result<()> {
     } else {
         // no user namespace but idmap
         if !linux.uid_mappings.is_empty() || !linux.gid_mappings.is_empty() {
-            return Err(anyhow!("No user namespace, but uid or gid mapping exists"));
+            return Err(einval());
         }
     }
 
@@ -167,7 +163,7 @@ fn sysctl(oci: &Spec) -> Result<()> {
             if contain_namespace(&linux.namespaces, "ipc") {
                 continue;
             } else {
-                return Err(anyhow!("Linux namespace does not contain ipc"));
+                return Err(einval());
             }
         }
 
@@ -182,11 +178,11 @@ fn sysctl(oci: &Spec) -> Result<()> {
             }
 
             if key == "kernel.hostname" {
-                return Err(anyhow!("Kernel hostname specfied in Spec"));
+                return Err(einval());
             }
         }
 
-        return Err(anyhow!("Sysctl config contains invalid settings"));
+        return Err(einval());
     }
     Ok(())
 }
@@ -195,13 +191,12 @@ fn rootless_euid_mapping(oci: &Spec) -> Result<()> {
     let linux = get_linux(oci)?;
 
     if !contain_namespace(&linux.namespaces, "user") {
-        return Err(anyhow!("Linux namespace is missing user"));
+        return Err(einval());
     }
 
     if linux.uid_mappings.is_empty() || linux.gid_mappings.is_empty() {
-        return Err(anyhow!(
-            "Rootless containers require at least one UID/GID mapping"
-        ));
+        // rootless containers requires at least one UID/GID mapping
+        return Err(einval());
     }
 
     Ok(())
@@ -225,7 +220,7 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                 let fields: Vec<&str> = opt.split('=').collect();
 
                 if fields.len() != 2 {
-                    return Err(anyhow!("Options has invalid field: {:?}", fields));
+                    return Err(einval());
                 }
 
                 let id = fields[1]
@@ -234,11 +229,11 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                     .context(format!("parse field {}", &fields[1]))?;
 
                 if opt.starts_with("uid=") && !has_idmapping(&linux.uid_mappings, id) {
-                    return Err(anyhow!("uid of {} does not have a valid mapping", id));
+                    return Err(einval());
                 }
 
                 if opt.starts_with("gid=") && !has_idmapping(&linux.gid_mappings, id) {
-                    return Err(anyhow!("gid of {} does not have a valid mapping", id));
+                    return Err(einval());
                 }
             }
         }
@@ -254,18 +249,15 @@ fn rootless_euid(oci: &Spec) -> Result<()> {
 
 pub fn validate(conf: &Config) -> Result<()> {
     lazy_static::initialize(&SYSCTLS);
-    let oci = conf
-        .spec
-        .as_ref()
-        .ok_or_else(|| anyhow!("Invalid config spec"))?;
+    let oci = conf.spec.as_ref().ok_or_else(einval)?;
 
     if oci.linux.is_none() {
-        return Err(anyhow!("oci Linux is none"));
+        return Err(einval());
     }
 
     let root = match oci.root.as_ref() {
         Some(v) => v.path.as_str(),
-        None => return Err(anyhow!("oci root is none")),
+        None => return Err(einval()),
     };
 
     rootfs(root).context("rootfs")?;

src/agent/src/config.rs

@@ -11,9 +11,6 @@ use std::fs;
 use std::str::FromStr;
 use std::time;
 use tracing::instrument;
-use url::Url;
-
-use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
 
 const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console";
 const DEV_MODE_FLAG: &str = "agent.devmode";
@@ -26,17 +23,12 @@ const LOG_VPORT_OPTION: &str = "agent.log_vport";
 const CONTAINER_PIPE_SIZE_OPTION: &str = "agent.container_pipe_size";
 const UNIFIED_CGROUP_HIERARCHY_OPTION: &str = "agent.unified_cgroup_hierarchy";
 const CONFIG_FILE: &str = "agent.config_file";
-const CONTAINER_POLICY_FILE: &str = "agent.container_policy_file";
-const AA_KBC_PARAMS: &str = "agent.aa_kbc_params";
-const HTTPS_PROXY: &str = "agent.https_proxy";
-const NO_PROXY: &str = "agent.no_proxy";
-const ENABLE_DATA_INTEGRITY: &str = "agent.data_integrity";
-const ENABLE_SIGNATURE_VERIFICATION: &str = "agent.enable_signature_verification";
 
 const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
 const DEFAULT_CONTAINER_PIPE_SIZE: i32 = 0;
 const VSOCK_ADDR: &str = "vsock://-1";
+const VSOCK_PORT: u16 = 1024;
 
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
@@ -59,11 +51,6 @@ const ERR_INVALID_CONTAINER_PIPE_SIZE_PARAM: &str = "unable to parse container p
 const ERR_INVALID_CONTAINER_PIPE_SIZE_KEY: &str = "invalid container pipe size key name";
 const ERR_INVALID_CONTAINER_PIPE_NEGATIVE: &str = "container pipe size should not be negative";
 
-const ERR_INVALID_CONTAINER_POLICY_PATH_VALUE: &str = "invalid container_policy_file value";
-const ERR_INVALID_CONTAINER_POLICY_PATH_KEY: &str = "invalid container_policy_file key";
-const ERR_INVALID_CONTAINER_POLICY_ABSOLUTE: &str =
-    "container_policy_file path must be an absolute file path";
-
 #[derive(Debug, Default, Deserialize)]
 pub struct EndpointsConfig {
     pub allowed: Vec<String>,
@@ -89,12 +76,6 @@ pub struct AgentConfig {
     pub tracing: bool,
     pub endpoints: AgentEndpoints,
     pub supports_seccomp: bool,
-    pub container_policy_path: String,
-    pub aa_kbc_params: String,
-    pub https_proxy: String,
-    pub no_proxy: String,
-    pub data_integrity: bool,
-    pub enable_signature_verification: bool,
 }
 
 #[derive(Debug, Deserialize)]
@@ -110,12 +91,6 @@ pub struct AgentConfigBuilder {
     pub unified_cgroup_hierarchy: Option<bool>,
     pub tracing: Option<bool>,
     pub endpoints: Option<EndpointsConfig>,
-    pub container_policy_path: Option<String>,
-    pub aa_kbc_params: Option<String>,
-    pub https_proxy: Option<String>,
-    pub no_proxy: Option<String>,
-    pub data_integrity: Option<bool>,
-    pub enable_signature_verification: Option<bool>,
 }
 
 macro_rules! config_override {
@@ -172,17 +147,11 @@ impl Default for AgentConfig {
             debug_console_vport: 0,
             log_vport: 0,
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-            server_addr: format!("{}:{}", VSOCK_ADDR, DEFAULT_AGENT_VSOCK_PORT),
+            server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
             unified_cgroup_hierarchy: false,
             tracing: false,
             endpoints: Default::default(),
             supports_seccomp: rpc::have_seccomp(),
-            container_policy_path: String::from(""),
-            aa_kbc_params: String::from(""),
-            https_proxy: String::from(""),
-            no_proxy: String::from(""),
-            data_integrity: false,
-            enable_signature_verification: true,
         }
     }
 }
@@ -211,16 +180,6 @@ impl FromStr for AgentConfig {
         config_override!(agent_config_builder, agent_config, server_addr);
         config_override!(agent_config_builder, agent_config, unified_cgroup_hierarchy);
         config_override!(agent_config_builder, agent_config, tracing);
-        config_override!(agent_config_builder, agent_config, container_policy_path);
-        config_override!(agent_config_builder, agent_config, aa_kbc_params);
-        config_override!(agent_config_builder, agent_config, https_proxy);
-        config_override!(agent_config_builder, agent_config, no_proxy);
-        config_override!(agent_config_builder, agent_config, data_integrity);
-        config_override!(
-            agent_config_builder,
-            agent_config,
-            enable_signature_verification
-        );
 
         // Populate the allowed endpoints hash set, if we got any from the config file.
         if let Some(endpoints) = agent_config_builder.endpoints {
@@ -249,10 +208,6 @@ impl AgentConfig {
         let mut config: AgentConfig = Default::default();
         let cmdline = fs::read_to_string(file)?;
         let params: Vec<&str> = cmdline.split_ascii_whitespace().collect();
-
-        let mut using_config_file = false;
-        // Check if there is config file before parsing params that might
-        // override values from the config file.
         for param in params.iter() {
             // If we get a configuration file path from the command line, we
             // generate our config from it.
@@ -260,13 +215,9 @@ impl AgentConfig {
             // or if it can't be parsed properly.
             if param.starts_with(format!("{}=", CONFIG_FILE).as_str()) {
                 let config_file = get_string_value(param)?;
-                config = AgentConfig::from_config_file(&config_file)?;
-                using_config_file = true;
-                break;
+                return AgentConfig::from_config_file(&config_file);
             }
-        }
 
-        for param in params.iter() {
             // parse cmdline flags
             parse_cmdline_param!(param, DEBUG_CONSOLE_FLAG, config.debug_console);
             parse_cmdline_param!(param, DEV_MODE_FLAG, config.dev_mode);
@@ -326,30 +277,6 @@ impl AgentConfig {
                 config.unified_cgroup_hierarchy,
                 get_bool_value
             );
-
-            parse_cmdline_param!(
-                param,
-                CONTAINER_POLICY_FILE,
-                config.container_policy_path,
-                get_container_policy_path_value
-            );
-
-            parse_cmdline_param!(param, AA_KBC_PARAMS, config.aa_kbc_params, get_string_value);
-            parse_cmdline_param!(param, HTTPS_PROXY, config.https_proxy, get_url_value);
-            parse_cmdline_param!(param, NO_PROXY, config.no_proxy, get_string_value);
-            parse_cmdline_param!(
-                param,
-                ENABLE_DATA_INTEGRITY,
-                config.data_integrity,
-                get_bool_value
-            );
-
-            parse_cmdline_param!(
-                param,
-                ENABLE_SIGNATURE_VERIFICATION,
-                config.enable_signature_verification,
-                get_bool_value
-            );
         }
 
         if let Ok(addr) = env::var(SERVER_ADDR_ENV_VAR) {
@@ -369,9 +296,7 @@ impl AgentConfig {
         }
 
         // We did not get a configuration file: allow all endpoints.
-        if !using_config_file {
-            config.endpoints.all_allowed = true;
-        }
+        config.endpoints.all_allowed = true;
 
         Ok(config)
     }
@@ -505,38 +430,9 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
     Ok(value)
 }
 
-#[instrument]
-fn get_container_policy_path_value(param: &str) -> Result<String> {
-    let fields: Vec<&str> = param.split('=').collect();
-
-    ensure!(!fields[0].is_empty(), ERR_INVALID_CONTAINER_POLICY_PATH_KEY);
-    ensure!(fields.len() == 2, ERR_INVALID_CONTAINER_POLICY_PATH_VALUE);
-
-    let key = fields[0];
-    ensure!(
-        key == CONTAINER_POLICY_FILE,
-        ERR_INVALID_CONTAINER_POLICY_PATH_KEY
-    );
-
-    let value = String::from(fields[1]);
-    ensure!(!value.is_empty(), ERR_INVALID_CONTAINER_POLICY_PATH_VALUE);
-    ensure!(
-        value.starts_with('/'),
-        ERR_INVALID_CONTAINER_POLICY_ABSOLUTE
-    );
-    ensure!(!value.contains(".."), ERR_INVALID_CONTAINER_POLICY_ABSOLUTE);
-    Ok(value)
-}
-
-#[instrument]
-fn get_url_value(param: &str) -> Result<String> {
-    let value = get_string_value(param)?;
-    Ok(Url::parse(&value)?.to_string())
-}
-
 #[cfg(test)]
 mod tests {
-    use test_utils::assert_result;
+    use crate::assert_result;
 
     use super::*;
     use anyhow::anyhow;
@@ -552,8 +448,6 @@ mod tests {
         assert!(!config.dev_mode);
         assert_eq!(config.log_level, DEFAULT_LOG_LEVEL);
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
-        assert_eq!(config.container_policy_path, "");
-        assert!(config.enable_signature_verification);
     }
 
     #[test]
@@ -572,12 +466,6 @@ mod tests {
             server_addr: &'a str,
             unified_cgroup_hierarchy: bool,
             tracing: bool,
-            container_policy_path: &'a str,
-            aa_kbc_params: &'a str,
-            https_proxy: &'a str,
-            no_proxy: &'a str,
-            data_integrity: bool,
-            enable_signature_verification: bool,
         }
 
         impl Default for TestData<'_> {
@@ -593,12 +481,6 @@ mod tests {
                     server_addr: TEST_SERVER_ADDR,
                     unified_cgroup_hierarchy: false,
                     tracing: false,
-                    container_policy_path: "",
-                    aa_kbc_params: "",
-                    https_proxy: "",
-                    no_proxy: "",
-                    data_integrity: false,
-                    enable_signature_verification: true,
                 }
             }
         }
@@ -968,86 +850,6 @@ mod tests {
                 tracing: true,
                 ..Default::default()
             },
-            TestData {
-                contents: "agent.container_policy_file=/etc/containers/policy.json",
-                container_policy_path: "/etc/containers/policy.json",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.aa_kbc_params=offline_fs_kbc::null",
-                aa_kbc_params: "offline_fs_kbc::null",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.aa_kbc_params=eaa_kbc::127.0.0.1:50000",
-                aa_kbc_params: "eaa_kbc::127.0.0.1:50000",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://proxy.url.com:81/",
-                https_proxy: "http://proxy.url.com:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.https_proxy=http://192.168.1.100:81/",
-                https_proxy: "http://192.168.1.100:81/",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=*.internal.url.com",
-                no_proxy: "*.internal.url.com",
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.no_proxy=192.168.1.0/24,172.16.0.0/12",
-                no_proxy: "192.168.1.0/24,172.16.0.0/12",
-                ..Default::default()
-            },
-            TestData {
-                contents: "",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=true",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=false",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=1",
-                data_integrity: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.data_integrity=0",
-                data_integrity: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=false",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=0",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=1",
-                enable_signature_verification: true,
-                ..Default::default()
-            },
-            TestData {
-                contents: "agent.enable_signature_verification=foo",
-                enable_signature_verification: false,
-                ..Default::default()
-            },
         ];
 
         let dir = tempdir().expect("failed to create tmpdir");
@@ -1095,20 +897,6 @@ mod tests {
             assert_eq!(d.container_pipe_size, config.container_pipe_size, "{}", msg);
             assert_eq!(d.server_addr, config.server_addr, "{}", msg);
             assert_eq!(d.tracing, config.tracing, "{}", msg);
-            assert_eq!(
-                d.container_policy_path, config.container_policy_path,
-                "{}",
-                msg
-            );
-            assert_eq!(d.aa_kbc_params, config.aa_kbc_params, "{}", msg);
-            assert_eq!(d.https_proxy, config.https_proxy, "{}", msg);
-            assert_eq!(d.no_proxy, config.no_proxy, "{}", msg);
-            assert_eq!(d.data_integrity, config.data_integrity, "{}", msg);
-            assert_eq!(
-                d.enable_signature_verification, config.enable_signature_verification,
-                "{}",
-                msg
-            );
 
             for v in vars_to_unset {
                 env::remove_var(v);
@@ -1580,72 +1368,6 @@ Caused by:
         }
     }
 
-    #[test]
-    fn test_get_container_policy_path_value() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            param: &'a str,
-            result: Result<String>,
-        }
-
-        let tests = &[
-            TestData {
-                param: "",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_PATH_KEY)),
-            },
-            TestData {
-                param: "agent.container_policy_file",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_PATH_VALUE)),
-            },
-            TestData {
-                param: "agent.container_policy_file=",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_PATH_VALUE)),
-            },
-            TestData {
-                param: "foo=bar",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_PATH_KEY)),
-            },
-            TestData {
-                param: "agent.policy_path=/another/absolute/path.json",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_PATH_KEY)),
-            },
-            TestData {
-                param: "agent.container_policy_file=/etc/container/policy.json",
-                result: Ok("/etc/container/policy.json".into()),
-            },
-            TestData {
-                param: "agent.container_policy_file=/another/absolute/path.json",
-                result: Ok("/another/absolute/path.json".into()),
-            },
-            TestData {
-                param: "agent.container_policy_file=./relative/path.json",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_ABSOLUTE)),
-            },
-            TestData {
-                param: "agent.container_policy_file=./relative/path.json",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_ABSOLUTE)),
-            },
-            TestData {
-                param: "agent.container_policy_file=../../relative/path.json",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_ABSOLUTE)),
-            },
-            TestData {
-                param: "agent.container_policy_file=junk_string",
-                result: Err(anyhow!(ERR_INVALID_CONTAINER_POLICY_ABSOLUTE)),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            let result = get_container_policy_path_value(d.param);
-
-            let msg = format!("{}: result: {:?}", msg, result);
-
-            assert_result!(d.result, result, msg);
-        }
-    }
-
     #[test]
     fn test_config_builder_from_string() {
         let config = AgentConfig::from_str(
@@ -1676,50 +1398,4 @@ Caused by:
         // Verify that the default values are valid
         assert_eq!(config.hotplug_timeout, DEFAULT_HOTPLUG_TIMEOUT);
     }
-
-    #[test]
-    fn test_config_from_cmdline_and_config_file() {
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        let agent_config = r#"
-               dev_mode = false
-               server_addr = 'vsock://8:2048'
-
-               [endpoints]
-               allowed = ["CreateContainer", "StartContainer"]
-              "#;
-
-        let config_path = dir.path().join("agent-config.toml");
-        let config_filename = config_path.to_str().expect("failed to get config filename");
-
-        fs::write(config_filename, agent_config).expect("failed to write agen config");
-
-        let cmdline = format!("agent.devmode agent.config_file={}", config_filename);
-
-        let cmdline_path = dir.path().join("cmdline");
-        let cmdline_filename = cmdline_path
-            .to_str()
-            .expect("failed to get cmdline filename");
-
-        fs::write(cmdline_filename, cmdline).expect("failed to write agen config");
-
-        let config = AgentConfig::from_cmdline(cmdline_filename, vec![])
-            .expect("failed to parse command line");
-
-        // Should be overwritten by cmdline
-        assert!(config.dev_mode);
-
-        // Should be from agent config
-        assert_eq!(config.server_addr, "vsock://8:2048");
-
-        // Should be from agent config
-        assert_eq!(
-            config.endpoints.allowed,
-            vec!["CreateContainer".to_string(), "StartContainer".to_string()]
-                .iter()
-                .cloned()
-                .collect()
-        );
-        assert!(!config.endpoints.all_allowed);
-    }
 }

src/agent/src/console.rs

@@ -9,7 +9,7 @@ use anyhow::{anyhow, Result};
 use nix::fcntl::{self, FcntlArg, FdFlag, OFlag};
 use nix::libc::{STDERR_FILENO, STDIN_FILENO, STDOUT_FILENO};
 use nix::pty::{openpty, OpenptyResult};
-use nix::sys::socket::{self, AddressFamily, SockFlag, SockType, VsockAddr};
+use nix::sys::socket::{self, AddressFamily, SockAddr, SockFlag, SockType};
 use nix::sys::stat::Mode;
 use nix::sys::wait;
 use nix::unistd::{self, close, dup2, fork, setsid, ForkResult, Pid};
@@ -67,7 +67,7 @@ pub async fn debug_console_handler(
             SockFlag::SOCK_CLOEXEC,
             None,
         )?;
-        let addr = VsockAddr::new(libc::VMADDR_CID_ANY, port);
+        let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, port);
         socket::bind(listenfd, &addr)?;
         socket::listen(listenfd, 1)?;
 

src/agent/src/device.rs

@@ -571,15 +571,13 @@ fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -
 
     if let Some(resources) = linux.resources.as_mut() {
         for r in &mut resources.devices {
-            if let (Some(host_type), Some(host_major), Some(host_minor)) =
-                (r.r#type.as_ref(), r.major, r.minor)
-            {
-                if let Some(update) = res_updates.get(&(host_type.as_str(), host_major, host_minor))
+            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
+                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
                 {
                     info!(
                         sl!(),
                         "update_spec_devices() updating resource";
-                        "type" => &host_type,
+                        "type" => &r.r#type,
                         "host_major" => host_major,
                         "host_minor" => host_minor,
                         "guest_major" => update.guest_major,
@@ -856,7 +854,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         allow: false,
         major: Some(major),
         minor: Some(minor),
-        r#type: Some(String::from("b")),
+        r#type: String::from("b"),
         access: String::from("rw"),
     });
 
@@ -1019,13 +1017,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_a),
                             minor: Some(host_minor_a),
                             ..oci::LinuxDeviceCgroup::default()
                         },
                         oci::LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major_b),
                             minor: Some(host_minor_b),
                             ..oci::LinuxDeviceCgroup::default()
@@ -1118,13 +1116,13 @@ mod tests {
                 resources: Some(LinuxResources {
                     devices: vec![
                         LinuxDeviceCgroup {
-                            r#type: Some("c".to_string()),
+                            r#type: "c".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()
                         },
                         LinuxDeviceCgroup {
-                            r#type: Some("b".to_string()),
+                            r#type: "b".to_string(),
                             major: Some(host_major),
                             minor: Some(host_minor),
                             ..LinuxDeviceCgroup::default()