Merge pull request #6282 from amshinde/3.0.2-branch-bump

# Kata Containers 3.0.2

.github/cargo-deny-composite-action/cargo-deny-generator.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Copyright (c) 2022 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+script_dir=$(dirname "$(readlink -f "$0")")
+parent_dir=$(realpath "${script_dir}/../..")
+cidir="${parent_dir}/ci"
+source "${cidir}/lib.sh"
+
+cargo_deny_file="${script_dir}/action.yaml"
+
+cat cargo-deny-skeleton.yaml.in > "${cargo_deny_file}"
+
+changed_files_status=$(run_get_pr_changed_file_details)
+changed_files_status=$(echo "$changed_files_status" | grep "Cargo\.toml$" || true)
+changed_files=$(echo "$changed_files_status" | awk '{print $NF}' || true)
+
+if [ -z "$changed_files" ]; then
+  cat >> "${cargo_deny_file}" << EOF
+    - run: echo "No Cargo.toml files to check"
+      shell: bash
+EOF
+fi
+
+for path in $changed_files
+do
+    cat >> "${cargo_deny_file}" << EOF
+
+    - name: ${path}
+      continue-on-error: true
+      shell: bash
+      run: |
+        pushd $(dirname ${path})
+        cargo deny check
+        popd
+EOF
+done

.github/cargo-deny-composite-action/cargo-deny-skeleton.yaml.in

@@ -0,0 +1,30 @@
+#
+# Copyright (c) 2022 Red Hat
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+name: 'Cargo Crates Check'
+description: 'Checks every Cargo.toml file using cargo-deny'
+
+env:
+  CARGO_TERM_COLOR: always
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Rust
+      uses: actions-rs/toolchain@v1
+      with:
+        profile: minimal
+        toolchain: nightly 
+        override: true
+
+    - name: Cache
+      uses: Swatinem/rust-cache@v2
+
+    - name: Install Cargo deny
+      shell: bash
+      run: |
+        which cargo
+        cargo install --locked cargo-deny || true

.github/workflows/add-backport-label.yaml

@@ -0,0 +1,100 @@
+name: Add backport label
+
+on:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - edited
+      - labeled
+      - unlabeled
+
+jobs:
+  check-issues:
+    if: ${{ github.event.label.name != 'auto-backport' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code to allow hub to communicate with the project
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/checkout@v3
+
+      - name: Install hub extension script
+        run: |
+          pushd $(mktemp -d) &>/dev/null
+          git clone --single-branch --depth 1 "https://github.com/kata-containers/.github" && cd .github/scripts
+          sudo install hub-util.sh /usr/local/bin
+          popd &>/dev/null
+
+      - name: Determine whether to add label 
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          CONTAINS_AUTO_BACKPORT: ${{ contains(github.event.pull_request.labels.*.name, 'auto-backport') }}
+        id: add_label
+        run: |
+          pr=${{ github.event.pull_request.number }}
+          linked_issue_urls=$(hub-util.sh \
+            list-issues-for-pr "$pr" |\
+            grep -v "^\#"  |\
+            cut -d';' -f3 || true)
+          [ -z "$linked_issue_urls" ] && {
+            echo "::error::No linked issues for PR $pr"
+            exit 1
+          }
+          has_bug=false
+          for issue_url in $(echo "$linked_issue_urls")
+          do
+            issue=$(echo "$issue_url"| awk -F\/ '{print $NF}' || true)
+            [ -z "$issue" ] && {
+              echo "::error::Cannot determine issue number from $issue_url for PR $pr"
+              exit 1
+            }
+            labels=$(hub-util.sh list-labels-for-issue "$issue")
+            
+            label_names=$(echo $labels | jq -r '.[].name' || true)
+            if [[ "$label_names" =~ "bug" ]]; then
+              has_bug=true
+              break
+            fi
+          done
+
+          has_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'needs-backport') }}
+          has_no_backport_needed_label=${{ contains(github.event.pull_request.labels.*.name, 'no-backport-needed') }}
+
+          echo "::set-output name=add_backport_label::false"
+          if [ $has_backport_needed_label  = true ] || [ $has_bug  = true ]; then
+            if [[ $has_no_backport_needed_label = false ]]; then
+              echo "::set-output name=add_backport_label::true"
+            fi
+          fi
+
+          # Do not spam comment, only if auto-backport label is going to be newly added.
+          echo "::set-output name=auto_backport_added::$CONTAINS_AUTO_BACKPORT"
+
+      - name: Add comment
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' && steps.add_label.outputs.auto_backport_added == 'false' }}
+        uses: actions/github-script@v6
+        with:
+          script: |
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: 'This issue has been marked for auto-backporting. Add label(s) backport-to-BRANCHNAME to backport to them'
+            })
+
+      # Allow label to be removed by adding no-backport-needed label
+      - name: Remove auto-backport label
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'false' }}
+        uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
+        with:
+          remove-labels: "auto-backport"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Add auto-backport label
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') && steps.add_label.outputs.add_backport_label == 'true' }}
+        uses: andymckay/labeler@e6c4322d0397f3240f0e7e30a33b5c5df2d39e90
+        with:
+          add-labels: "auto-backport"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/auto-backport.yaml

@@ -0,0 +1,29 @@
+on:
+  pull_request_target:
+    types: ["labeled", "closed"]
+
+jobs:
+  backport:
+    name: Backport PR
+    runs-on: ubuntu-latest
+    if: |
+      github.event.pull_request.merged == true
+      && contains(github.event.pull_request.labels.*.name, 'auto-backport')
+      && (
+        (github.event.action == 'labeled' && github.event.label.name == 'auto-backport')
+        || (github.event.action == 'closed')
+      )
+    steps:
+      - name: Backport Action
+        uses: sqren/backport-github-action@v8.9.2
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          auto_backport_label_prefix: backport-to-
+
+      - name: Info log
+        if: ${{ success() }}
+        run: cat /home/runner/.backport/backport.info.log
+        
+      - name: Debug log
+        if: ${{ failure() }}
+        run: cat /home/runner/.backport/backport.debug.log

.github/workflows/cargo-deny-runner.yaml

@@ -0,0 +1,19 @@
+name: Cargo Crates Check Runner
+on: [pull_request]
+jobs:
+  cargo-deny-runner:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Code
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: actions/checkout@v3
+      - name: Generate Action
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        run: bash cargo-deny-generator.sh
+        working-directory: ./.github/cargo-deny-composite-action/
+        env:
+          GOPATH: ${{ runner.workspace }}/kata-containers
+      - name: Run Action
+        if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+        uses: ./.github/cargo-deny-composite-action

.github/workflows/darwin-tests.yaml

@@ -9,16 +9,12 @@ on:
 name: Darwin tests
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [macos-latest]
-    runs-on: ${{ matrix.os }}
+    runs-on: macos-latest
     steps:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.19.2
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Build utils

.github/workflows/docs-url-alive-check.yaml

@@ -5,11 +5,7 @@ on:
 name: Docs URL Alive Check
 jobs:
   test:
-    strategy:
-      matrix:
-        go-version: [1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-20.04
     # don't run this action on forks
     if: github.repository_owner == 'kata-containers'
     env:
@@ -18,7 +14,7 @@ jobs:
     - name: Install Go
       uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.19.2
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Set env

.github/workflows/kata-deploy-push.yaml

@@ -25,6 +25,7 @@ jobs:
           - rootfs-image
           - rootfs-initrd
           - virtiofsd
+          - nydus
     steps:
       - uses: actions/checkout@v2
       - name: Install docker

.github/workflows/kata-deploy-test.yaml

@@ -44,6 +44,7 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - nydus
           - qemu
           - rootfs-image
           - rootfs-initrd

.github/workflows/release.yaml

@@ -13,6 +13,7 @@ jobs:
           - cloud-hypervisor
           - firecracker
           - kernel
+          - nydus
           - qemu
           - rootfs-image
           - rootfs-initrd

.github/workflows/static-checks.yaml

@@ -8,12 +8,8 @@ on:
 
 name: Static checks
 jobs:
-  test:
-    strategy:
-      matrix:
-        go-version: [1.16.x, 1.17.x]
-        os: [ubuntu-20.04]
-    runs-on: ${{ matrix.os }}
+  check-vendored-code:
+    runs-on: ubuntu-20.04
     env:
       TRAVIS: "true"
       TRAVIS_BRANCH: ${{ github.base_ref }}
@@ -24,9 +20,9 @@ jobs:
     steps:
     - name: Install Go
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@v3
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.19.2
       env:
         GOPATH: ${{ runner.workspace }}/kata-containers
     - name: Setup GOPATH
@@ -43,7 +39,59 @@ jobs:
         echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
     - name: Checkout code
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
+        target_branch=${TRAVIS_BRANCH}
+    - name: Setup
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    # Check whether the vendored code is up-to-date & working as the first thing
+    - name: Check vendored code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
+
+  static-checks:
+    runs-on: ubuntu-20.04
+    env:
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+      RUST_BACKTRACE: "1"
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.2
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Setup GOPATH
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         path: ./src/github.com/${{ github.repository }}
@@ -73,23 +121,205 @@ jobs:
         echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
         echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
         echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
-    # Check whether the vendored code is up-to-date & working as the first thing
-    - name: Check vendored code
-      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
-      run: |
-        cd ${GOPATH}/src/github.com/${{ github.repository }} && make vendor
     - name: Static Checks
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make static-checks
+
+
+  compiler-checks:
+    runs-on: ubuntu-20.04
+    env:
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+      RUST_BACKTRACE: "1"
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.2
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Setup GOPATH
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
+        target_branch=${TRAVIS_BRANCH}
+    - name: Setup
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Installing rust
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
+        PATH=$PATH:"$HOME/.cargo/bin"
+        rustup target add x86_64-unknown-linux-musl
+        rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Run Compiler Checks
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make check
+
+  unit-tests:
+    runs-on: ubuntu-20.04
+    env:
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+      RUST_BACKTRACE: "1"
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.2
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Setup GOPATH
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
+        target_branch=${TRAVIS_BRANCH}
+    - name: Setup
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Installing rust
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
+        PATH=$PATH:"$HOME/.cargo/bin"
+        rustup target add x86_64-unknown-linux-musl
+        rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Run Unit Tests
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |
         cd ${GOPATH}/src/github.com/${{ github.repository }} && make test
+
+  unit-tests-as-root:
+    runs-on: ubuntu-20.04
+    env:
+      TRAVIS: "true"
+      TRAVIS_BRANCH: ${{ github.base_ref }}
+      TRAVIS_PULL_REQUEST_BRANCH: ${{ github.head_ref }}
+      TRAVIS_PULL_REQUEST_SHA : ${{ github.event.pull_request.head.sha }}
+      RUST_BACKTRACE: "1"
+      target_branch: ${{ github.base_ref }}
+    steps:
+    - name: Install Go
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.19.2 
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Setup GOPATH
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH: ${TRAVIS_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_BRANCH: ${TRAVIS_PULL_REQUEST_BRANCH}"
+        echo "TRAVIS_PULL_REQUEST_SHA: ${TRAVIS_PULL_REQUEST_SHA}"
+        echo "TRAVIS: ${TRAVIS}"
+    - name: Set env
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
+        echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
+    - name: Checkout code
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        path: ./src/github.com/${{ github.repository }}
+    - name: Setup travis references
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        echo "TRAVIS_BRANCH=${TRAVIS_BRANCH:-$(echo $GITHUB_REF | awk 'BEGIN { FS = \"/\" } ; { print $3 }')}"
+        target_branch=${TRAVIS_BRANCH}
+    - name: Setup
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/setup.sh
+      env:
+        GOPATH: ${{ runner.workspace }}/kata-containers
+    - name: Installing rust
+      if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
+      run: |
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_rust.sh
+        PATH=$PATH:"$HOME/.cargo/bin"
+        rustup target add x86_64-unknown-linux-musl
+        rustup component add rustfmt clippy
+    - name: Setup seccomp
+      run: |
+        libseccomp_install_dir=$(mktemp -d -t libseccomp.XXXXXXXXXX)
+        gperf_install_dir=$(mktemp -d -t gperf.XXXXXXXXXX)
+        cd ${GOPATH}/src/github.com/${{ github.repository }} && ./ci/install_libseccomp.sh "${libseccomp_install_dir}" "${gperf_install_dir}"
+        echo "Set environment variables for the libseccomp crate to link the libseccomp library statically"
+        echo "LIBSECCOMP_LINK_TYPE=static" >> $GITHUB_ENV
+        echo "LIBSECCOMP_LIB_PATH=${libseccomp_install_dir}/lib" >> $GITHUB_ENV
     - name: Run Unit Tests As Root User
       if: ${{ !contains(github.event.pull_request.labels.*.name, 'force-skip-ci') }}
       run: |

README.md

@@ -117,6 +117,7 @@ The table below lists the core parts of the project:
 | Component | Type | Description |
 |-|-|-|
 | [runtime](src/runtime) | core | Main component run by a container manager and providing a containerd shimv2 runtime implementation. |
+| [runtime-rs](src/runtime-rs) | core | The Rust version runtime. |
 | [agent](src/agent) | core | Management process running inside the virtual machine / POD that sets up the container environment. |
 | [libraries](src/libs) | core | Library crates shared by multiple Kata Container components or published to [`crates.io`](https://crates.io/index.html) |
 | [`dragonball`](src/dragonball) | core | An optional built-in VMM brings out-of-the-box Kata Containers experience with optimizations on container workloads |

VERSION

@@ -1 +1 @@
-3.0.0-alpha0
+3.0.2

ci/install_libseccomp.sh

@@ -23,25 +23,27 @@ arch=${ARCH:-$(uname -m)}
 workdir="$(mktemp -d --tmpdir build-libseccomp.XXXXX)"
 
 # Variables for libseccomp
-# Currently, specify the libseccomp version directly without using `versions.yaml`
-# because the current Snap workflow is incomplete.
-# After solving the issue, replace this code by using the `versions.yaml`.
-# libseccomp_version=$(get_version "externals.libseccomp.version")
-# libseccomp_url=$(get_version "externals.libseccomp.url")
-libseccomp_version="2.5.1"
-libseccomp_url="https://github.com/seccomp/libseccomp"
+libseccomp_version="${LIBSECCOMP_VERSION:-""}"
+if [ -z "${libseccomp_version}" ]; then
+    libseccomp_version=$(get_version "externals.libseccomp.version")
+fi
+libseccomp_url="${LIBSECCOMP_URL:-""}"
+if [ -z "${libseccomp_url}" ]; then
+    libseccomp_url=$(get_version "externals.libseccomp.url")
+fi
 libseccomp_tarball="libseccomp-${libseccomp_version}.tar.gz"
 libseccomp_tarball_url="${libseccomp_url}/releases/download/v${libseccomp_version}/${libseccomp_tarball}"
 cflags="-O2"
 
 # Variables for gperf
-# Currently, specify the gperf version directly without using `versions.yaml`
-# because the current Snap workflow is incomplete.
-# After solving the issue, replace this code by using the `versions.yaml`.
-# gperf_version=$(get_version "externals.gperf.version")
-# gperf_url=$(get_version "externals.gperf.url")
-gperf_version="3.1"
-gperf_url="https://ftp.gnu.org/gnu/gperf"
+gperf_version="${GPERF_VERSION:-""}"
+if [ -z "${gperf_version}" ]; then
+    gperf_version=$(get_version "externals.gperf.version")
+fi
+gperf_url="${GPERF_URL:-""}"
+if [ -z "${gperf_url}" ]; then
+    gperf_url=$(get_version "externals.gperf.url")
+fi
 gperf_tarball="gperf-${gperf_version}.tar.gz"
 gperf_tarball_url="${gperf_url}/${gperf_tarball}"
 

ci/lib.sh

@@ -54,3 +54,13 @@ run_docs_url_alive_check()
 	git fetch -a
 	bash "$tests_repo_dir/.ci/static-checks.sh" --docs --all "github.com/kata-containers/kata-containers"
 }
+
+run_get_pr_changed_file_details()
+{
+	clone_tests_repo
+	# Make sure we have the targeting branch
+	git remote set-branches --add origin "${branch}"
+	git fetch -a
+	source "$tests_repo_dir/.ci/lib.sh"
+	get_pr_changed_file_details
+}

deny.toml

@@ -0,0 +1,33 @@
+targets = [
+    { triple = "x86_64-apple-darwin" },
+    { triple = "x86_64-unknown-linux-gnu" },
+    { triple = "x86_64-unknown-linux-musl" },
+]
+
+[advisories]
+vulnerability = "deny"
+unsound = "deny"
+unmaintained = "deny"
+ignore = ["RUSTSEC-2020-0071"]
+
+[bans]
+multiple-versions = "allow"
+deny = [
+    { name = "cmake" },
+    { name = "openssl-sys" },
+]
+
+[licenses]
+unlicensed = "deny"
+allow-osi-fsf-free = "neither"
+copyleft = "allow"
+# We want really high confidence when inferring licenses from text
+confidence-threshold = 0.93
+allow = ["0BSD", "Apache-2.0", "BSD-2-Clause", "BSD-3-Clause", "CC0-1.0", "ISC", "MIT", "MPL-2.0"]
+private = { ignore = true}
+
+exceptions = []
+
+[sources]
+unknown-registry = "allow"
+unknown-git = "allow"

docs/Developer-Guide.md

@@ -522,7 +522,7 @@ bash-4.2# exit
 exit
 ```
 
-`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/master/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
+`kata-runtime exec` has a command-line option `runtime-namespace`, which is used to specify under which [runtime namespace](https://github.com/containerd/containerd/blob/main/docs/namespaces.md) the particular pod was created. By default, it is set to `k8s.io` and works for containerd when configured
  with Kubernetes. For CRI-O, the namespace should set to `default` explicitly. This should not be confused with [Kubernetes namespaces](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/).
 For other CRI-runtimes and configurations, you may need to set the namespace utilizing the `runtime-namespace` option.
 

docs/Unit-Test-Advice.md

@@ -341,7 +341,7 @@ The main repository has the most comprehensive set of skip abilities. See:
 
 One method is to use the `nix` crate along with some custom macros:
 
-```
+```rust
 #[cfg(test)]
 mod tests {
     #[allow(unused_macros)]

docs/design/architecture_3.0/README.md

@@ -81,31 +81,31 @@ In our case, there will be a variety of resources, and every resource has severa
 - Stage 2 (September): support common features
 - Stage 3: support full features
 
-| **Class**                  | **Sub-Class**       | **Development Stage** |
-| -------------------------- | ------------------- | --------------------- |
-| Service                    | task service        | Stage 1               |
-|                            | extend service      | Stage 3               |
-|                            | image service       | Stage 3               |
-| Runtime handler            | `Virt-Container`    | Stage 1               |
-|                            | `Wasm-Container`    | Stage 3               |
-|                            | `Linux-Container`   | Stage 3               |
-| Endpoint                   | VETH Endpoint       | Stage 1               |
-|                            | Physical Endpoint   | Stage 2               |
-|                            | Tap Endpoint        | Stage 2               |
-|                            | `Tuntap` Endpoint   | Stage 2               |
-|                            | `IPVlan` Endpoint   | Stage 3               |
-|                            | `MacVlan` Endpoint  | Stage 3               |
-|                            | MACVTAP Endpoint    | Stage 3               |
-|                            | `VhostUserEndpoint` | Stage 3               |
-| Network Interworking Model | Tc filter           | Stage 1               |
-|                            | `MacVtap`           | Stage 3               |
-| Storage                    | Virtio-fs           | Stage 1               |
-|                            | `nydus`             | Stage 2               |
-| Hypervisor                 | `Dragonball`        | Stage 1               |
-|                            | QEMU                | Stage 2               |
-|                            | ACRN                | Stage 3               |
-|                            | Cloud Hypervisor    | Stage 3               |
-|                            | Firecracker         | Stage 3               |
+| **Class**                  | **Sub-Class**       | **Development Stage** | **Status** |
+| -------------------------- | ------------------- | --------------------- |------------|
+| Service                    | task service        | Stage 1               |  ✅        |
+|                            | extend service      | Stage 3               |  🚫        |
+|                            | image service       | Stage 3               |  🚫        |
+| Runtime handler            | `Virt-Container`    | Stage 1               |  ✅        |
+| Endpoint                   | VETH Endpoint       | Stage 1               |  ✅        |
+|                            | Physical Endpoint   | Stage 2               |  ✅        |
+|                            | Tap Endpoint        | Stage 2               |  ✅        |
+|                            | `Tuntap` Endpoint   | Stage 2               |  ✅        |
+|                            | `IPVlan` Endpoint   | Stage 2               |  ✅        |
+|                            | `MacVlan` Endpoint  | Stage 2               |  ✅        |
+|                            | MACVTAP Endpoint    | Stage 3               |  🚫        |
+|                            | `VhostUserEndpoint` | Stage 3               |  🚫        |
+| Network Interworking Model | Tc filter           | Stage 1               |  ✅        |
+|                            | `MacVtap`           | Stage 3               |  🚧        |
+| Storage                    | Virtio-fs           | Stage 1               |  ✅        |
+|                            | `nydus`             | Stage 2               |  🚧        |
+|                            | `device mapper`     | Stage 2               |  🚫        |
+| `Cgroup V2`                |                     | Stage 2               |  🚧        |
+| Hypervisor                 | `Dragonball`        | Stage 1               |  🚧        |
+|                            | QEMU                | Stage 2               |  🚫        |
+|                            | ACRN                | Stage 3               |  🚫        |
+|                            | Cloud Hypervisor    | Stage 3               |  🚫        |
+|                            | Firecracker         | Stage 3               |  🚫        |
 
 ## FAQ
 

docs/how-to/how-to-hotplug-memory-arm64.md

@@ -15,6 +15,18 @@ $ sudo .ci/aarch64/install_rom_aarch64.sh
 $ popd
 ```
 
+## Config KATA QEMU
+
+After executing the above script, two files will be generated under the directory `/usr/share/kata-containers/` by default, namely `kata-flash0.img` and `kata-flash1.img`. Next we need to change the configuration file of `kata qemu`, which is in `/opt/kata/share/defaults/kata-containers/configuration-qemu.toml` by default, specify in the configuration file to use the UEFI ROM installed above. The above is an example of `kata deploy` installation. For package management installation, please use `kata-runtime env` to find the location of the configuration file. Please refer to the following configuration.
+
+```
+[hypervisor.qemu]
+
+# -pflash can add image file to VM. The arguments of it should be in format
+# of ["/path/to/flash0.img", "/path/to/flash1.img"]
+pflashes = ["/usr/share/kata-containers/kata-flash0.img", "/usr/share/kata-containers/kata-flash1.img"]
+```
+
 ## Run for test
 
 Let's test if the memory hotplug is ready for Kata after install the UEFI ROM. Make sure containerd is ready to run Kata before test.

docs/install/minikube-installation-guide.md

@@ -55,11 +55,11 @@ Here are the features to set up a CRI-O based Minikube, and why you need them:
 
 | what | why |
 | ---- | --- |
-| `--bootstrapper=kubeadm` | As recommended for [minikube CRI-o](https://kubernetes.io/docs/setup/minikube/#cri-o) |
+| `--bootstrapper=kubeadm` | As recommended for [minikube CRI-O](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) |
 | `--container-runtime=cri-o` | Using CRI-O for Kata |
-| `--enable-default-cni` | As recommended for [minikube CRI-o](https://kubernetes.io/docs/setup/minikube/#cri-o) |
+| `--enable-default-cni` | As recommended for [minikube CRI-O](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) |
 | `--memory 6144` | Allocate sufficient memory, as Kata Containers default to 1 or 2Gb |
-| `--network-plugin=cni` | As recommended for [minikube CRI-o](https://kubernetes.io/docs/setup/minikube/#cri-o) |
+| `--network-plugin=cni` | As recommended for [minikube CRI-O](https://minikube.sigs.k8s.io/docs/handbook/config/#runtime-configuration) |
 | `--vm-driver kvm2` | The host VM driver |
 
 To use containerd, modify the `--container-runtime` argument:

snap/snapcraft.yaml

@@ -82,8 +82,39 @@ parts:
       fi
       rustup component add rustfmt
 
+  docker:
+    after: [metadata]
+    plugin: nil
+    prime:
+      - -*
+    build-packages:
+      - ca-certificates
+      - containerd
+      - curl
+      - gnupg
+      - lsb-release
+      - runc
+    override-build: |
+      source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
+
+      curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
+          sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+      distro_codename=$(lsb_release -cs)
+      echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
+          sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
+      sudo apt-get -y update
+      sudo apt-get -y install docker-ce docker-ce-cli containerd.io
+
+      echo "Unmasking docker service"
+      sudo -E systemctl unmask docker.service || true
+      sudo -E systemctl unmask docker.socket || true
+      echo "Adding $USER into docker group"
+      sudo -E gpasswd -a $USER docker
+      echo "Starting docker"
+      sudo -E systemctl start docker || true
+
   image:
-    after: [godeps, qemu, kernel]
+    after: [godeps, docker, qemu, kernel]
     plugin: nil
     build-packages:
       - docker.io
@@ -107,14 +138,6 @@ parts:
       # Copy yq binary. It's used in the container
       cp -a "${yq}" "${GOPATH}/bin/"
 
-      echo "Unmasking docker service"
-      sudo -E systemctl unmask docker.service || true
-      sudo -E systemctl unmask docker.socket || true
-      echo "Adding $USER into docker group"
-      sudo -E gpasswd -a $USER docker
-      echo "Starting docker"
-      sudo -E systemctl start docker || true
-
       cd "${kata_dir}/tools/osbuilder"
 
       # build image
@@ -206,7 +229,7 @@ parts:
 
       # Install raw kernel
       vmlinux_path="vmlinux"
-      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/compressed/vmlinux"
+      [ "${arch}" = "s390x" ] && vmlinux_path="arch/s390/boot/vmlinux"
       vmlinux_name="vmlinux-${kernel_suffix}"
       cp "${vmlinux_path}" "${kata_kernel_dir}/${vmlinux_name}"
       ln -sf "${vmlinux_name}" "${kata_kernel_dir}/vmlinux.container"
@@ -301,54 +324,31 @@ parts:
 
   virtiofsd:
     plugin: nil
-    after: [godeps, rustdeps]
+    after: [godeps, rustdeps, docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
-      # Currently, powerpc makes use of the QEMU's C implementation.
-      # The other platforms make use of the new rust virtiofsd.
-      #
-      # See "tools/packaging/scripts/configure-hypervisor.sh".
-      if [ "${arch}" == "ppc64le" ]
-      then
-          echo "INFO: Building QEMU's C version of virtiofsd"
-          # Handled by the 'qemu' part, so nothing more to do here.
-          exit 0
-      else
-          echo "INFO: Building rust version of virtiofsd"
-      fi
+      echo "INFO: Building rust version of virtiofsd"
 
-      cd "${kata_dir}"
+      cd "${SNAPCRAFT_PROJECT_DIR}"
+      # Clean-up build dir in case it already exists
+      sudo -E NO_TTY=true make virtiofsd-tarball
 
-      export PATH=${PATH}:${HOME}/.cargo/bin
-      # Download the rust implementation of virtiofsd
-      tools/packaging/static-build/virtiofsd/build-static-virtiofsd.sh
       sudo install \
         --owner='root' \
         --group='root' \
         --mode=0755 \
         -D \
         --target-directory="${SNAPCRAFT_PART_INSTALL}/usr/libexec/" \
-        virtiofsd/virtiofsd
+        build/virtiofsd/builddir/virtiofsd/virtiofsd
 
   cloud-hypervisor:
     plugin: nil
-    after: [godeps]
+    after: [godeps, docker]
     override-build: |
       source "${SNAPCRAFT_PROJECT_DIR}/snap/local/snap-common.sh"
 
       if [ "${arch}" == "aarch64" ] || [ "${arch}" == "x86_64" ]; then
-          sudo apt-get -y update
-          sudo apt-get -y install ca-certificates curl gnupg lsb-release
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg |\
-              sudo gpg --batch --yes --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-          distro_codename=$(lsb_release -cs)
-          echo "deb [arch=${dpkg_arch} signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu ${distro_codename} stable" |\
-              sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-          sudo apt-get -y update
-          sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-          sudo systemctl start docker.socket
-
           cd "${SNAPCRAFT_PROJECT_DIR}"
           sudo -E NO_TTY=true make cloud-hypervisor-tarball
 

src/agent/Cargo.lock

@@ -17,6 +17,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "ansi_term"
 version = "0.12.1"
@@ -168,26 +177,28 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "cgroups-rs"
-version = "0.2.9"
+version = "0.2.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cdae996d9638ba03253ffa1c93345a585974a97abbdeab9176c77922f3efc1e8"
+checksum = "cf5525f2cf84d5113ab26bfb6474180eb63224b4b1e4be31ee87be4098f11399"
 dependencies = [
  "libc",
  "log",
- "nix 0.23.1",
+ "nix 0.24.2",
  "regex",
 ]
 
 [[package]]
 name = "chrono"
-version = "0.4.19"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
 dependencies = [
- "libc",
+ "iana-time-zone",
+ "js-sys",
  "num-integer",
  "num-traits",
  "time 0.1.44",
+ "wasm-bindgen",
  "winapi",
 ]
 
@@ -529,6 +540,19 @@ version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "js-sys",
+ "wasm-bindgen",
+ "winapi",
+]
+
 [[package]]
 name = "indexmap"
 version = "1.9.1"
@@ -625,12 +649,13 @@ dependencies = [
  "futures",
  "ipnetwork",
  "kata-sys-util",
+ "kata-types",
  "lazy_static",
  "libc",
  "log",
  "logging",
- "netlink-packet-utils 0.4.1",
- "netlink-sys 0.7.0",
+ "netlink-packet-utils",
+ "netlink-sys",
  "nix 0.24.2",
  "oci",
  "opentelemetry",
@@ -651,6 +676,7 @@ dependencies = [
  "slog-stdlog",
  "sysinfo",
  "tempfile",
+ "test-utils",
  "thiserror",
  "tokio",
  "tokio-vsock",
@@ -660,6 +686,7 @@ dependencies = [
  "tracing-subscriber",
  "ttrpc",
  "vsock-exporter",
+ "which",
 ]
 
 [[package]]
@@ -717,21 +744,20 @@ checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
 
 [[package]]
 name = "libseccomp"
-version = "0.1.3"
+version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36ad71a5b66ceef3acfe6a3178b29b4da063f8bcb2c36dab666d52a7a9cfdb86"
+checksum = "49bda1fbf25c42ac8942ff7df1eb6172a3bc36299e84be0dba8c888a7db68c80"
 dependencies = [
  "libc",
  "libseccomp-sys",
- "nix 0.17.0",
  "pkg-config",
 ]
 
 [[package]]
 name = "libseccomp-sys"
-version = "0.1.1"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "539912de229a4fc16e507e8df12a394038a524a5b5b6c92045ad344472aac475"
+checksum = "9a7cbbd4ad467251987c6e5b47d53b11a5a05add08f2447a9e2d70aef1e0d138"
 
 [[package]]
 name = "lock_api"
@@ -816,28 +842,28 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
 [[package]]
 name = "netlink-packet-core"
-version = "0.4.2"
+version = "0.2.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "345b8ab5bd4e71a2986663e88c56856699d060e78e152e6e9d7966fcd5491297"
+checksum = "ac48279d5062bdf175bdbcb6b58ff1d6b0ecd54b951f7a0ff4bc0550fe903ccb"
 dependencies = [
  "anyhow",
  "byteorder",
  "libc",
- "netlink-packet-utils 0.5.1",
+ "netlink-packet-utils",
 ]
 
 [[package]]
 name = "netlink-packet-route"
-version = "0.13.0"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5dee5ed749373c298237fe694eb0a51887f4cc1a27370c8464bac4382348f1a"
+checksum = "76aed5d3b6e3929713bf1e1334a11fd65180b6d9f5d7c8572664c48b122604f8"
 dependencies = [
  "anyhow",
  "bitflags",
  "byteorder",
  "libc",
  "netlink-packet-core",
- "netlink-packet-utils 0.5.1",
+ "netlink-packet-utils",
 ]
 
 [[package]]
@@ -852,31 +878,19 @@ dependencies = [
  "thiserror",
 ]
 
-[[package]]
-name = "netlink-packet-utils"
-version = "0.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "25af9cf0dc55498b7bd94a1508af7a78706aa0ab715a73c5169273e03c84845e"
-dependencies = [
- "anyhow",
- "byteorder",
- "paste",
- "thiserror",
-]
-
 [[package]]
 name = "netlink-proto"
-version = "0.10.0"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "65b4b14489ab424703c092062176d52ba55485a89c076b4f9db05092b7223aa6"
+checksum = "ddd06e90449ae973fe3888c1ff85949604ef5189b4ac9a2ae39518da1e00762d"
 dependencies = [
  "bytes 1.1.0",
  "futures",
  "log",
  "netlink-packet-core",
- "netlink-sys 0.8.3",
- "thiserror",
+ "netlink-sys",
  "tokio",
+ "tokio-util",
 ]
 
 [[package]]
@@ -891,30 +905,17 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "netlink-sys"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92b654097027250401127914afb37cb1f311df6610a9891ff07a757e94199027"
-dependencies = [
- "bytes 1.1.0",
- "futures",
- "libc",
- "log",
- "tokio",
-]
-
 [[package]]
 name = "nix"
-version = "0.17.0"
+version = "0.22.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "50e4785f2c3b7589a0d0c1dd60285e1188adac4006e8abd6dd578e1567027363"
+checksum = "e4916f159ed8e5de0082076562152a76b7a1f64a01fd9d1e0fea002c37624faf"
 dependencies = [
  "bitflags",
  "cc",
- "cfg-if 0.1.10",
+ "cfg-if 1.0.0",
  "libc",
- "void",
+ "memoffset",
 ]
 
 [[package]]
@@ -1479,15 +1480,15 @@ dependencies = [
 
 [[package]]
 name = "rtnetlink"
-version = "0.11.0"
+version = "0.8.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "46f1cfa18f8cebe685373a2697915d7e0db3b4554918bba118385e0f71f258a7"
+checksum = "7c9a6200d18ec1acfc218ce71363dcc9b6075f399220f903fdfeacd476a876ef"
 dependencies = [
  "futures",
  "log",
  "netlink-packet-route",
  "netlink-proto",
- "nix 0.24.2",
+ "nix 0.22.3",
  "thiserror",
  "tokio",
 ]
@@ -1507,7 +1508,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "libseccomp",
- "nix 0.23.1",
+ "nix 0.24.2",
  "oci",
  "path-absolutize",
  "protobuf",
@@ -1523,6 +1524,7 @@ dependencies = [
  "slog",
  "slog-scope",
  "tempfile",
+ "test-utils",
  "tokio",
 ]
 
@@ -1769,6 +1771,13 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "test-utils"
+version = "0.1.0"
+dependencies = [
+ "nix 0.24.2",
+]
+
 [[package]]
 name = "textwrap"
 version = "0.15.0"
@@ -1868,6 +1877,20 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-util"
+version = "0.6.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
+dependencies = [
+ "bytes 1.1.0",
+ "futures-core",
+ "futures-sink",
+ "log",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-vsock"
 version = "0.3.1"
@@ -2049,12 +2072,6 @@ version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
 
-[[package]]
-name = "void"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
-
 [[package]]
 name = "vsock"
 version = "0.2.6"
@@ -2073,7 +2090,7 @@ dependencies = [
  "bincode",
  "byteorder",
  "libc",
- "nix 0.23.1",
+ "nix 0.24.2",
  "opentelemetry",
  "serde",
  "slog",
@@ -2156,13 +2173,13 @@ checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be"
 
 [[package]]
 name = "which"
-version = "4.2.5"
+version = "4.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae"
+checksum = "1c831fbbee9e129a8cf93e7747a82da9d95ba8e16621cae60ec2cdc849bacb7b"
 dependencies = [
  "either",
- "lazy_static",
  "libc",
+ "once_cell",
 ]
 
 [[package]]

src/agent/Cargo.toml

@@ -3,6 +3,7 @@ name = "kata-agent"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 oci = { path = "../libs/oci" }
@@ -12,15 +13,16 @@ lazy_static = "1.3.0"
 ttrpc = { version = "0.6.0", features = ["async"], default-features = false }
 protobuf = "2.27.0"
 libc = "0.2.58"
-nix = "0.24.1"
+nix = "0.24.2"
 capctl = "0.2.0"
 serde_json = "1.0.39"
 scan_fmt = "0.2.3"
 scopeguard = "1.0.0"
 thiserror = "1.0.26"
-regex = "1.5.5"
+regex = "1.5.6"
 serial_test = "0.5.1"
 kata-sys-util = { path = "../libs/kata-sys-util" }
+kata-types = { path = "../libs/kata-types" }
 sysinfo = "0.23.0"
 
 # Async helpers
@@ -33,7 +35,7 @@ tokio = { version = "1.14.0", features = ["full"] }
 tokio-vsock = "0.3.1"
 
 netlink-sys = { version = "0.7.0", features = ["tokio_socket",]}
-rtnetlink = "0.11.0"
+rtnetlink = "0.8.0"
 netlink-packet-utils = "0.4.1"
 ipnetwork = "0.17.0"
 
@@ -50,7 +52,7 @@ log = "0.4.11"
 prometheus = { version = "0.13.0", features = ["process"] }
 procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.8" }
+cgroups = { package = "cgroups-rs", version = "0.2.10" }
 
 # Tracing
 tracing = "0.1.26"
@@ -66,6 +68,8 @@ clap = { version = "3.0.1", features = ["derive"] }
 
 [dev-dependencies]
 tempfile = "3.1.0"
+test-utils = { path = "../libs/test-utils" }
+which = "4.3.0"
 
 [workspace]
 members = [

src/agent/rustjail/Cargo.toml

@@ -3,6 +3,7 @@ name = "rustjail"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 serde = "1.0.91"
@@ -11,7 +12,7 @@ serde_derive = "1.0.91"
 oci = { path = "../../libs/oci" }
 protocols = { path ="../../libs/protocols" }
 caps = "0.5.0"
-nix = "0.23.0"
+nix = "0.24.2"
 scopeguard = "1.0.0"
 capctl = "0.2.0"
 lazy_static = "1.3.0"
@@ -20,10 +21,10 @@ protobuf = "2.27.0"
 slog = "2.5.2"
 slog-scope = "4.1.2"
 scan_fmt = "0.2.6"
-regex = "1.5.5"
+regex = "1.5.6"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.8" }
+cgroups = { package = "cgroups-rs", version = "0.2.10" }
 rlimit = "0.5.3"
 cfg-if = "0.1.0"
 
@@ -31,11 +32,12 @@ tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "
 futures = "0.3.17"
 async-trait = "0.1.31"
 inotify = "0.9.2"
-libseccomp = { version = "0.1.3", optional = true }
+libseccomp = { version = "0.2.3", optional = true }
 
 [dev-dependencies]
 serial_test = "0.5.0"
 tempfile = "3.1.0"
+test-utils = { path = "../../libs/test-utils" }
 
 [features]
 seccomp = ["libseccomp"]

src/agent/rustjail/src/cgroups/fs/mod.rs

@@ -174,7 +174,7 @@ impl CgroupManager for Manager {
                 freezer_controller.freeze()?;
             }
             _ => {
-                return Err(anyhow!(nix::Error::EINVAL));
+                return Err(anyhow!("Invalid FreezerState"));
             }
         }
 

src/agent/rustjail/src/console.rs

@@ -6,8 +6,9 @@
 use anyhow::{anyhow, Result};
 use nix::errno::Errno;
 use nix::pty;
-use nix::sys::{socket, uio};
+use nix::sys::socket;
 use nix::unistd::{self, dup2};
+use std::io::IoSlice;
 use std::os::unix::io::{AsRawFd, RawFd};
 use std::path::Path;
 
@@ -23,10 +24,7 @@ pub fn setup_console_socket(csocket_path: &str) -> Result<Option<RawFd>> {
         None,
     )?;
 
-    match socket::connect(
-        socket_fd,
-        &socket::SockAddr::Unix(socket::UnixAddr::new(Path::new(csocket_path))?),
-    ) {
+    match socket::connect(socket_fd, &socket::UnixAddr::new(Path::new(csocket_path))?) {
         Ok(()) => Ok(Some(socket_fd)),
         Err(errno) => Err(anyhow!("failed to open console fd: {}", errno)),
     }
@@ -36,11 +34,11 @@ pub fn setup_master_console(socket_fd: RawFd) -> Result<()> {
     let pseudo = pty::openpty(None, None)?;
 
     let pty_name: &[u8] = b"/dev/ptmx";
-    let iov = [uio::IoVec::from_slice(pty_name)];
+    let iov = [IoSlice::new(pty_name)];
     let fds = [pseudo.master];
     let cmsg = socket::ControlMessage::ScmRights(&fds);
 
-    socket::sendmsg(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
+    socket::sendmsg::<()>(socket_fd, &iov, &[cmsg], socket::MsgFlags::empty(), None)?;
 
     unistd::setsid()?;
     let ret = unsafe { libc::ioctl(pseudo.slave, libc::TIOCSCTTY) };

src/agent/rustjail/src/container.rs

@@ -106,6 +106,11 @@ impl Default for ContainerStatus {
     }
 }
 
+// We might want to change this to thiserror in the future
+const MissingCGroupManager: &str = "failed to get container's cgroup Manager";
+const MissingLinux: &str = "no linux config";
+const InvalidNamespace: &str = "invalid namespace type";
+
 pub type Config = CreateOpts;
 type NamespaceType = String;
 
@@ -292,7 +297,7 @@ impl Container for LinuxContainer {
             self.status.transition(ContainerState::Paused);
             return Ok(());
         }
-        Err(anyhow!("failed to get container's cgroup manager"))
+        Err(anyhow!(MissingCGroupManager))
     }
 
     fn resume(&mut self) -> Result<()> {
@@ -310,7 +315,7 @@ impl Container for LinuxContainer {
             self.status.transition(ContainerState::Running);
             return Ok(());
         }
-        Err(anyhow!("failed to get container's cgroup manager"))
+        Err(anyhow!(MissingCGroupManager))
     }
 }
 
@@ -397,7 +402,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     };
 
     if spec.linux.is_none() {
-        return Err(anyhow!("no linux config"));
+        return Err(anyhow!(MissingLinux));
     }
     let linux = spec.linux.as_ref().unwrap();
 
@@ -411,7 +416,7 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     for ns in &nses {
         let s = NAMESPACES.get(&ns.r#type.as_str());
         if s.is_none() {
-            return Err(anyhow!("invalid ns type"));
+            return Err(anyhow!(InvalidNamespace));
         }
         let s = s.unwrap();
 
@@ -1437,18 +1442,10 @@ impl LinuxContainer {
             Some(unistd::getuid()),
             Some(unistd::getgid()),
         )
-        .context(format!("cannot change onwer of container {} root", id))?;
-
-        if config.spec.is_none() {
-            return Err(anyhow!(nix::Error::EINVAL));
-        }
+        .context(format!("Cannot change owner of container {} root", id))?;
 
         let spec = config.spec.as_ref().unwrap();
 
-        if spec.linux.is_none() {
-            return Err(anyhow!(nix::Error::EINVAL));
-        }
-
         let linux = spec.linux.as_ref().unwrap();
 
         let cpath = if linux.cgroups_path.is_empty() {
@@ -1457,7 +1454,12 @@ impl LinuxContainer {
             linux.cgroups_path.clone()
         };
 
-        let cgroup_manager = FsManager::new(cpath.as_str())?;
+        let cgroup_manager = FsManager::new(cpath.as_str()).map_err(|e| {
+            anyhow!(format!(
+                "fail to create cgroup manager with path {}: {:}",
+                cpath, e
+            ))
+        })?;
         info!(logger, "new cgroup_manager {:?}", &cgroup_manager);
 
         Ok(LinuxContainer {
@@ -1525,7 +1527,7 @@ pub async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()
     let binary = PathBuf::from(h.path.as_str());
     let path = binary.canonicalize()?;
     if !path.exists() {
-        return Err(anyhow!(nix::Error::EINVAL));
+        return Err(anyhow!("Path {:?} does not exist", path));
     }
 
     let mut args = h.args.clone();
@@ -1656,12 +1658,12 @@ fn valid_env(e: &str) -> Option<(&str, &str)> {
 mod tests {
     use super::*;
     use crate::process::Process;
-    use crate::skip_if_not_root;
     use nix::unistd::Uid;
     use std::fs;
     use std::os::unix::fs::MetadataExt;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
+    use test_utils::skip_if_not_root;
     use tokio::process::Command;
 
     macro_rules! sl {

src/agent/rustjail/src/lib.rs

@@ -514,15 +514,6 @@ pub fn grpc_to_oci(grpc: &grpc::Spec) -> oci::Spec {
 #[cfg(test)]
 mod tests {
     use super::*;
-    #[macro_export]
-    macro_rules! skip_if_not_root {
-        () => {
-            if !nix::unistd::Uid::effective().is_root() {
-                println!("INFO: skipping {} which needs root", module_path!());
-                return;
-            }
-        };
-    }
 
     // Parameters:
     //

src/agent/rustjail/src/mount.rs

@@ -1020,9 +1020,7 @@ pub fn finish_rootfs(cfd_log: RawFd, spec: &Spec, process: &Process) -> Result<(
 }
 
 fn mask_path(path: &str) -> Result<()> {
-    if !path.starts_with('/') || path.contains("..") {
-        return Err(anyhow!(nix::Error::EINVAL));
-    }
+    check_paths(path)?;
 
     match mount(
         Some("/dev/null"),
@@ -1040,9 +1038,7 @@ fn mask_path(path: &str) -> Result<()> {
 }
 
 fn readonly_path(path: &str) -> Result<()> {
-    if !path.starts_with('/') || path.contains("..") {
-        return Err(anyhow!(nix::Error::EINVAL));
-    }
+    check_paths(path)?;
 
     if let Err(e) = mount(
         Some(&path[1..]),
@@ -1068,11 +1064,20 @@ fn readonly_path(path: &str) -> Result<()> {
     Ok(())
 }
 
+fn check_paths(path: &str) -> Result<()> {
+    if !path.starts_with('/') || path.contains("..") {
+        return Err(anyhow!(
+            "Cannot mount {} (path does not start with '/' or contains '..').",
+            path
+        ));
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
     use crate::assert_result;
-    use crate::skip_if_not_root;
     use std::fs::create_dir;
     use std::fs::create_dir_all;
     use std::fs::remove_dir_all;
@@ -1080,6 +1085,7 @@ mod tests {
     use std::os::unix::fs;
     use std::os::unix::io::AsRawFd;
     use tempfile::tempdir;
+    use test_utils::skip_if_not_root;
 
     #[test]
     #[serial(chdir)]
@@ -1420,6 +1426,55 @@ mod tests {
         }
     }
 
+    #[test]
+    fn test_check_paths() {
+        #[derive(Debug)]
+        struct TestData<'a> {
+            name: &'a str,
+            path: &'a str,
+            result: Result<()>,
+        }
+
+        let tests = &[
+            TestData {
+                name: "valid path",
+                path: "/foo/bar",
+                result: Ok(()),
+            },
+            TestData {
+                name: "does not starts with /",
+                path: "foo/bar",
+                result: Err(anyhow!(
+                    "Cannot mount foo/bar (path does not start with '/' or contains '..')."
+                )),
+            },
+            TestData {
+                name: "contains ..",
+                path: "../foo/bar",
+                result: Err(anyhow!(
+                    "Cannot mount ../foo/bar (path does not start with '/' or contains '..')."
+                )),
+            },
+        ];
+
+        for (i, d) in tests.iter().enumerate() {
+            let msg = format!("test[{}]: {:?}", i, d.name);
+
+            let result = check_paths(d.path);
+
+            let msg = format!("{}: result: {:?}", msg, result);
+
+            if d.result.is_ok() {
+                assert!(result.is_ok());
+                continue;
+            }
+
+            let expected_error = format!("{}", d.result.as_ref().unwrap_err());
+            let actual_error = format!("{}", result.unwrap_err());
+            assert!(actual_error == expected_error, "{}", msg);
+        }
+    }
+
     #[test]
     fn test_check_proc_mount() {
         let mount = oci::Mount {

src/agent/rustjail/src/seccomp.rs

@@ -26,12 +26,15 @@ fn get_rule_conditions(args: &[LinuxSeccompArg]) -> Result<Vec<ScmpArgCompare>>
             return Err(anyhow!("seccomp opreator is required"));
         }
 
-        let cond = ScmpArgCompare::new(
-            arg.index,
-            ScmpCompareOp::from_str(&arg.op)?,
-            arg.value,
-            Some(arg.value_two),
-        );
+        let mut op = ScmpCompareOp::from_str(&arg.op)?;
+        let mut value = arg.value;
+        // For SCMP_CMP_MASKED_EQ, arg.value is the mask and arg.value_two is the value
+        if op == ScmpCompareOp::MaskedEqual(u64::default()) {
+            op = ScmpCompareOp::MaskedEqual(arg.value);
+            value = arg.value_two;
+        }
+
+        let cond = ScmpArgCompare::new(arg.index, op, value);
 
         conditions.push(cond);
     }
@@ -44,7 +47,7 @@ pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
 
     for syscall in &scmp.syscalls {
         for name in &syscall.names {
-            if get_syscall_from_name(name, None).is_err() {
+            if ScmpSyscall::from_name(name).is_err() {
                 unknown_syscalls.push(name.to_string());
             }
         }
@@ -60,7 +63,7 @@ pub fn get_unknown_syscalls(scmp: &LinuxSeccomp) -> Option<Vec<String>> {
 // init_seccomp creates a seccomp filter and loads it for the current process
 // including all the child processes.
 pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
-    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as u32))?;
+    let def_action = ScmpAction::from_str(scmp.default_action.as_str(), Some(libc::EPERM as i32))?;
 
     // Create a new filter context
     let mut filter = ScmpFilterContext::new_filter(def_action)?;
@@ -72,7 +75,7 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
     }
 
     // Unset no new privileges bit
-    filter.set_no_new_privs_bit(false)?;
+    filter.set_ctl_nnp(false)?;
 
     // Add a rule for each system call
     for syscall in &scmp.syscalls {
@@ -80,13 +83,13 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
             return Err(anyhow!("syscall name is required"));
         }
 
-        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret))?;
+        let action = ScmpAction::from_str(&syscall.action, Some(syscall.errno_ret as i32))?;
         if action == def_action {
             continue;
         }
 
         for name in &syscall.names {
-            let syscall_num = match get_syscall_from_name(name, None) {
+            let syscall_num = match ScmpSyscall::from_name(name) {
                 Ok(num) => num,
                 Err(_) => {
                     // If we cannot resolve the given system call, we assume it is not supported
@@ -96,10 +99,10 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
             };
 
             if syscall.args.is_empty() {
-                filter.add_rule(action, syscall_num, None)?;
+                filter.add_rule(action, syscall_num)?;
             } else {
                 let conditions = get_rule_conditions(&syscall.args)?;
-                filter.add_rule(action, syscall_num, Some(&conditions))?;
+                filter.add_rule_conditional(action, syscall_num, &conditions)?;
             }
         }
     }
@@ -119,10 +122,10 @@ pub fn init_seccomp(scmp: &LinuxSeccomp) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::skip_if_not_root;
     use libc::{dup3, process_vm_readv, EPERM, O_CLOEXEC};
     use std::io::Error;
     use std::ptr::null;
+    use test_utils::skip_if_not_root;
 
     macro_rules! syscall_assert {
         ($e1: expr, $e2: expr) => {

src/agent/rustjail/src/validator.rs

@@ -4,17 +4,15 @@
 //
 
 use crate::container::Config;
-use anyhow::{anyhow, Context, Error, Result};
+use anyhow::{anyhow, Context, Result};
 use oci::{Linux, LinuxIdMapping, LinuxNamespace, Spec};
 use std::collections::HashMap;
 use std::path::{Component, PathBuf};
 
-fn einval() -> Error {
-    anyhow!(nix::Error::EINVAL)
-}
-
 fn get_linux(oci: &Spec) -> Result<&Linux> {
-    oci.linux.as_ref().ok_or_else(einval)
+    oci.linux
+        .as_ref()
+        .ok_or_else(|| anyhow!("Unable to get Linux section from Spec"))
 }
 
 fn contain_namespace(nses: &[LinuxNamespace], key: &str) -> bool {
@@ -31,7 +29,10 @@ fn rootfs(root: &str) -> Result<()> {
     let path = PathBuf::from(root);
     // not absolute path or not exists
     if !path.exists() || !path.is_absolute() {
-        return Err(einval());
+        return Err(anyhow!(
+            "Path from {:?} does not exist or is not absolute",
+            root
+        ));
     }
 
     // symbolic link? ..?
@@ -49,7 +50,7 @@ fn rootfs(root: &str) -> Result<()> {
         if let Some(v) = c.as_os_str().to_str() {
             stack.push(v.to_string());
         } else {
-            return Err(einval());
+            return Err(anyhow!("Invalid path component (unable to convert to str)"));
         }
     }
 
@@ -58,10 +59,13 @@ fn rootfs(root: &str) -> Result<()> {
         cleaned.push(e);
     }
 
-    let canon = path.canonicalize().context("canonicalize")?;
+    let canon = path.canonicalize().context("failed to canonicalize path")?;
     if cleaned != canon {
         // There is symbolic in path
-        return Err(einval());
+        return Err(anyhow!(
+            "There may be illegal symbols in the path name. Cleaned ({:?}) and canonicalized ({:?}) paths do not match",
+            cleaned,
+            canon));
     }
 
     Ok(())
@@ -74,7 +78,7 @@ fn hostname(oci: &Spec) -> Result<()> {
 
     let linux = get_linux(oci)?;
     if !contain_namespace(&linux.namespaces, "uts") {
-        return Err(einval());
+        return Err(anyhow!("Linux namespace does not contain uts"));
     }
 
     Ok(())
@@ -88,7 +92,7 @@ fn security(oci: &Spec) -> Result<()> {
     }
 
     if !contain_namespace(&linux.namespaces, "mount") {
-        return Err(einval());
+        return Err(anyhow!("Linux namespace does not contain mount"));
     }
 
     // don't care about selinux at present
@@ -103,7 +107,7 @@ fn idmapping(maps: &[LinuxIdMapping]) -> Result<()> {
         }
     }
 
-    Err(einval())
+    Err(anyhow!("No idmap has size > 0"))
 }
 
 fn usernamespace(oci: &Spec) -> Result<()> {
@@ -121,7 +125,7 @@ fn usernamespace(oci: &Spec) -> Result<()> {
     } else {
         // no user namespace but idmap
         if !linux.uid_mappings.is_empty() || !linux.gid_mappings.is_empty() {
-            return Err(einval());
+            return Err(anyhow!("No user namespace, but uid or gid mapping exists"));
         }
     }
 
@@ -163,7 +167,7 @@ fn sysctl(oci: &Spec) -> Result<()> {
             if contain_namespace(&linux.namespaces, "ipc") {
                 continue;
             } else {
-                return Err(einval());
+                return Err(anyhow!("Linux namespace does not contain ipc"));
             }
         }
 
@@ -178,11 +182,11 @@ fn sysctl(oci: &Spec) -> Result<()> {
             }
 
             if key == "kernel.hostname" {
-                return Err(einval());
+                return Err(anyhow!("Kernel hostname specfied in Spec"));
             }
         }
 
-        return Err(einval());
+        return Err(anyhow!("Sysctl config contains invalid settings"));
     }
     Ok(())
 }
@@ -191,12 +195,13 @@ fn rootless_euid_mapping(oci: &Spec) -> Result<()> {
     let linux = get_linux(oci)?;
 
     if !contain_namespace(&linux.namespaces, "user") {
-        return Err(einval());
+        return Err(anyhow!("Linux namespace is missing user"));
     }
 
     if linux.uid_mappings.is_empty() || linux.gid_mappings.is_empty() {
-        // rootless containers requires at least one UID/GID mapping
-        return Err(einval());
+        return Err(anyhow!(
+            "Rootless containers require at least one UID/GID mapping"
+        ));
     }
 
     Ok(())
@@ -220,7 +225,7 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                 let fields: Vec<&str> = opt.split('=').collect();
 
                 if fields.len() != 2 {
-                    return Err(einval());
+                    return Err(anyhow!("Options has invalid field: {:?}", fields));
                 }
 
                 let id = fields[1]
@@ -229,11 +234,11 @@ fn rootless_euid_mount(oci: &Spec) -> Result<()> {
                     .context(format!("parse field {}", &fields[1]))?;
 
                 if opt.starts_with("uid=") && !has_idmapping(&linux.uid_mappings, id) {
-                    return Err(einval());
+                    return Err(anyhow!("uid of {} does not have a valid mapping", id));
                 }
 
                 if opt.starts_with("gid=") && !has_idmapping(&linux.gid_mappings, id) {
-                    return Err(einval());
+                    return Err(anyhow!("gid of {} does not have a valid mapping", id));
                 }
             }
         }
@@ -249,15 +254,18 @@ fn rootless_euid(oci: &Spec) -> Result<()> {
 
 pub fn validate(conf: &Config) -> Result<()> {
     lazy_static::initialize(&SYSCTLS);
-    let oci = conf.spec.as_ref().ok_or_else(einval)?;
+    let oci = conf
+        .spec
+        .as_ref()
+        .ok_or_else(|| anyhow!("Invalid config spec"))?;
 
     if oci.linux.is_none() {
-        return Err(einval());
+        return Err(anyhow!("oci Linux is none"));
     }
 
     let root = match oci.root.as_ref() {
         Some(v) => v.path.as_str(),
-        None => return Err(einval()),
+        None => return Err(anyhow!("oci root is none")),
     };
 
     rootfs(root).context("rootfs")?;

src/agent/src/config.rs

@@ -12,6 +12,8 @@ use std::str::FromStr;
 use std::time;
 use tracing::instrument;
 
+use kata_types::config::default::DEFAULT_AGENT_VSOCK_PORT;
+
 const DEBUG_CONSOLE_FLAG: &str = "agent.debug_console";
 const DEV_MODE_FLAG: &str = "agent.devmode";
 const TRACE_MODE_OPTION: &str = "agent.trace";
@@ -28,7 +30,6 @@ const DEFAULT_LOG_LEVEL: slog::Level = slog::Level::Info;
 const DEFAULT_HOTPLUG_TIMEOUT: time::Duration = time::Duration::from_secs(3);
 const DEFAULT_CONTAINER_PIPE_SIZE: i32 = 0;
 const VSOCK_ADDR: &str = "vsock://-1";
-const VSOCK_PORT: u16 = 1024;
 
 // Environment variables used for development and testing
 const SERVER_ADDR_ENV_VAR: &str = "KATA_AGENT_SERVER_ADDR";
@@ -147,7 +148,7 @@ impl Default for AgentConfig {
             debug_console_vport: 0,
             log_vport: 0,
             container_pipe_size: DEFAULT_CONTAINER_PIPE_SIZE,
-            server_addr: format!("{}:{}", VSOCK_ADDR, VSOCK_PORT),
+            server_addr: format!("{}:{}", VSOCK_ADDR, DEFAULT_AGENT_VSOCK_PORT),
             unified_cgroup_hierarchy: false,
             tracing: false,
             endpoints: Default::default(),
@@ -432,7 +433,7 @@ fn get_container_pipe_size(param: &str) -> Result<i32> {
 
 #[cfg(test)]
 mod tests {
-    use crate::assert_result;
+    use test_utils::assert_result;
 
     use super::*;
     use anyhow::anyhow;

src/agent/src/main.rs

@@ -49,8 +49,6 @@ mod pci;
 pub mod random;
 mod sandbox;
 mod signal;
-#[cfg(test)]
-mod test_utils;
 mod uevent;
 mod util;
 mod version;
@@ -401,7 +399,8 @@ use std::os::unix::io::{FromRawFd, RawFd};
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::test_utils::TestUserType;
+    use test_utils::TestUserType;
+    use test_utils::{assert_result, skip_if_not_root, skip_if_root};
 
     #[tokio::test]
     async fn test_create_logger_task() {

src/agent/src/mount.rs

@@ -169,11 +169,12 @@ pub fn baremount(
 
     info!(
         logger,
-        "mount source={:?}, dest={:?}, fs_type={:?}, options={:?}",
+        "baremount source={:?}, dest={:?}, fs_type={:?}, options={:?}, flags={:?}",
         source,
         destination,
         fs_type,
-        options
+        options,
+        flags
     );
 
     nix::mount::mount(
@@ -778,8 +779,20 @@ pub async fn add_storages(
             }
         };
 
-        // Todo need to rollback the mounted storage if err met.
-        let mount_point = res?;
+        let mount_point = match res {
+            Err(e) => {
+                error!(
+                    logger,
+                    "add_storages failed, storage: {:?}, error: {:?} ", storage, e
+                );
+                let mut sb = sandbox.lock().await;
+                sb.unset_sandbox_storage(&storage.mount_point)
+                    .map_err(|e| warn!(logger, "fail to unset sandbox storage {:?}", e))
+                    .ok();
+                return Err(e);
+            }
+            Ok(m) => m,
+        };
 
         if !mount_point.is_empty() {
             mount_list.push(mount_point);
@@ -840,7 +853,8 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul
         return Err(anyhow!("Invalid mount point {}", mount_point));
     }
 
-    let content = fs::read_to_string(mount_file)?;
+    let content = fs::read_to_string(mount_file)
+        .map_err(|e| anyhow!("read mount file {}: {}", mount_file, e))?;
 
     let re = Regex::new(format!("device .+ mounted on {} with fstype (.+)", mount_point).as_str())?;
 
@@ -1016,8 +1030,6 @@ fn parse_options(option_list: Vec<String>) -> HashMap<String, String> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::test_utils::test_utils::TestUserType;
-    use crate::{skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root};
     use protobuf::RepeatedField;
     use protocols::agent::FSGroup;
     use std::fs::File;
@@ -1025,6 +1037,10 @@ mod tests {
     use std::io::Write;
     use std::path::PathBuf;
     use tempfile::tempdir;
+    use test_utils::TestUserType;
+    use test_utils::{
+        skip_if_not_root, skip_loop_by_user, skip_loop_if_not_root, skip_loop_if_root,
+    };
 
     #[test]
     fn test_mount() {

src/agent/src/namespace.rs

@@ -187,9 +187,10 @@ impl fmt::Debug for NamespaceType {
 #[cfg(test)]
 mod tests {
     use super::{Namespace, NamespaceType};
-    use crate::{mount::remove_mounts, skip_if_not_root};
+    use crate::mount::remove_mounts;
     use nix::sched::CloneFlags;
     use tempfile::Builder;
+    use test_utils::skip_if_not_root;
 
     #[tokio::test]
     async fn test_setup_persistent_ns() {

src/agent/src/netlink.rs

@@ -4,7 +4,7 @@
 //
 
 use anyhow::{anyhow, Context, Result};
-use futures::{future, TryStreamExt};
+use futures::{future, StreamExt, TryStreamExt};
 use ipnetwork::{IpNetwork, Ipv4Network, Ipv6Network};
 use nix::errno::Errno;
 use protobuf::RepeatedField;
@@ -164,7 +164,7 @@ impl Handle {
         let request = self.handle.link().get();
 
         let filtered = match filter {
-            LinkFilter::Name(name) => request.match_name(name.to_owned()),
+            LinkFilter::Name(name) => request.set_name_filter(name.to_owned()),
             LinkFilter::Index(index) => request.match_index(index),
             _ => request, // Post filters
         };
@@ -516,24 +516,70 @@ impl Handle {
     }
 
     /// Adds an ARP neighbor.
+    /// TODO: `rtnetlink` has no neighbours API, remove this after https://github.com/little-dude/netlink/pull/135
     async fn add_arp_neighbor(&mut self, neigh: &ARPNeighbor) -> Result<()> {
         let ip_address = neigh
             .toIPAddress
             .as_ref()
             .map(|to| to.address.as_str()) // Extract address field
             .and_then(|addr| if addr.is_empty() { None } else { Some(addr) }) // Make sure it's not empty
-            .ok_or_else(|| anyhow!(nix::Error::EINVAL))?;
+            .ok_or_else(|| anyhow!("Unable to determine ip address of ARP neighbor"))?;
 
         let ip = IpAddr::from_str(ip_address)
             .map_err(|e| anyhow!("Failed to parse IP {}: {:?}", ip_address, e))?;
 
+        // Import rtnetlink objects that make sense only for this function
+        use packet::constants::{NDA_UNSPEC, NLM_F_ACK, NLM_F_CREATE, NLM_F_EXCL, NLM_F_REQUEST};
+        use packet::neighbour::{NeighbourHeader, NeighbourMessage};
+        use packet::nlas::neighbour::Nla;
+        use packet::{NetlinkMessage, NetlinkPayload, RtnlMessage};
+        use rtnetlink::Error;
+
+        const IFA_F_PERMANENT: u16 = 0x80; // See https://github.com/little-dude/netlink/blob/0185b2952505e271805902bf175fee6ea86c42b8/netlink-packet-route/src/rtnl/constants.rs#L770
+
         let link = self.find_link(LinkFilter::Name(&neigh.device)).await?;
 
-        self.handle
-            .neighbours()
-            .add(link.index(), ip)
-            .execute()
-            .await?;
+        let message = NeighbourMessage {
+            header: NeighbourHeader {
+                family: match ip {
+                    IpAddr::V4(_) => packet::AF_INET,
+                    IpAddr::V6(_) => packet::AF_INET6,
+                } as u8,
+                ifindex: link.index(),
+                state: if neigh.state != 0 {
+                    neigh.state as u16
+                } else {
+                    IFA_F_PERMANENT
+                },
+                flags: neigh.flags as u8,
+                ntype: NDA_UNSPEC as u8,
+            },
+            nlas: {
+                let mut nlas = vec![Nla::Destination(match ip {
+                    IpAddr::V4(v4) => v4.octets().to_vec(),
+                    IpAddr::V6(v6) => v6.octets().to_vec(),
+                })];
+
+                if !neigh.lladdr.is_empty() {
+                    nlas.push(Nla::LinkLocalAddress(
+                        parse_mac_address(&neigh.lladdr)?.to_vec(),
+                    ));
+                }
+
+                nlas
+            },
+        };
+
+        // Send request and ACK
+        let mut req = NetlinkMessage::from(RtnlMessage::NewNeighbour(message));
+        req.header.flags = NLM_F_REQUEST | NLM_F_ACK | NLM_F_EXCL | NLM_F_CREATE;
+
+        let mut response = self.handle.request(req)?;
+        while let Some(message) = response.next().await {
+            if let NetlinkPayload::Error(err) = message.payload {
+                return Err(anyhow!(Error::NetlinkError(err)));
+            }
+        }
 
         Ok(())
     }
@@ -566,7 +612,12 @@ fn parse_mac_address(addr: &str) -> Result<[u8; 6]> {
 
     // Parse single Mac address block
     let mut parse_next = || -> Result<u8> {
-        let v = u8::from_str_radix(split.next().ok_or_else(|| anyhow!(nix::Error::EINVAL))?, 16)?;
+        let v = u8::from_str_radix(
+            split
+                .next()
+                .ok_or_else(|| anyhow!("Invalid MAC address {}", addr))?,
+            16,
+        )?;
         Ok(v)
     };
 
@@ -724,10 +775,10 @@ impl Address {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::skip_if_not_root;
     use rtnetlink::packet;
     use std::iter;
     use std::process::Command;
+    use test_utils::skip_if_not_root;
 
     #[tokio::test]
     async fn find_link_by_name() {
@@ -904,7 +955,7 @@ mod tests {
             .expect("prepare: failed to delete neigh");
     }
 
-    fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str, mac: &str) {
+    fn prepare_env_for_test_add_one_arp_neighbor(dummy_name: &str, ip: &str) {
         clean_env_for_test_add_one_arp_neighbor(dummy_name, ip);
         // modprobe dummy
         Command::new("modprobe")
@@ -918,12 +969,6 @@ mod tests {
             .output()
             .expect("failed to add dummy interface");
 
-        // ip link set dummy address 6a:92:3a:59:70:aa
-        Command::new("ip")
-            .args(&["link", "set", dummy_name, "address", mac])
-            .output()
-            .expect("failed to add dummy interface");
-
         // ip addr add 192.168.0.2/16 dev dummy
         Command::new("ip")
             .args(&["addr", "add", "192.168.0.2/16", "dev", dummy_name])
@@ -945,7 +990,7 @@ mod tests {
         let to_ip = "169.254.1.1";
         let dummy_name = "dummy_for_arp";
 
-        prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip, mac);
+        prepare_env_for_test_add_one_arp_neighbor(dummy_name, to_ip);
 
         let mut ip_address = IPAddress::new();
         ip_address.set_address(to_ip.to_string());

src/agent/src/network.rs

@@ -76,11 +76,11 @@ fn do_setup_guest_dns(logger: Logger, dns_list: Vec<String>, src: &str, dst: &st
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::skip_if_not_root;
     use nix::mount;
     use std::fs::File;
     use std::io::Write;
     use tempfile::tempdir;
+    use test_utils::skip_if_not_root;
 
     #[test]
     fn test_setup_guest_dns() {

src/agent/src/random.rs

@@ -53,9 +53,9 @@ pub fn reseed_rng(data: &[u8]) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::skip_if_not_root;
     use std::fs::File;
     use std::io::prelude::*;
+    use test_utils::skip_if_not_root;
 
     #[test]
     fn test_reseed_rng() {

src/agent/src/rpc.rs

@@ -226,7 +226,20 @@ impl AgentService {
             info!(sl!(), "no process configurations!");
             return Err(anyhow!(nix::Error::EINVAL));
         };
-        ctr.start(p).await?;
+
+        // if starting container failed, we will do some rollback work
+        // to ensure no resources are leaked.
+        if let Err(err) = ctr.start(p).await {
+            error!(sl!(), "failed to start container: {:?}", err);
+            if let Err(e) = ctr.destroy().await {
+                error!(sl!(), "failed to destroy container: {:?}", e);
+            }
+            if let Err(e) = remove_container_resources(&mut s, &cid) {
+                error!(sl!(), "failed to remove container resources: {:?}", e);
+            }
+            return Err(err);
+        }
+
         s.update_shared_pidns(&ctr)?;
         s.add_container(ctr);
         info!(sl!(), "created container!");
@@ -272,27 +285,6 @@ impl AgentService {
         req: protocols::agent::RemoveContainerRequest,
     ) -> Result<()> {
         let cid = req.container_id.clone();
-        let mut cmounts: Vec<String> = vec![];
-
-        let mut remove_container_resources = |sandbox: &mut Sandbox| -> Result<()> {
-            // Find the sandbox storage used by this container
-            let mounts = sandbox.container_mounts.get(&cid);
-            if let Some(mounts) = mounts {
-                for m in mounts.iter() {
-                    if sandbox.storages.get(m).is_some() {
-                        cmounts.push(m.to_string());
-                    }
-                }
-            }
-
-            for m in cmounts.iter() {
-                sandbox.unset_and_remove_sandbox_storage(m)?;
-            }
-
-            sandbox.container_mounts.remove(cid.as_str());
-            sandbox.containers.remove(cid.as_str());
-            Ok(())
-        };
 
         if req.timeout == 0 {
             let s = Arc::clone(&self.sandbox);
@@ -306,7 +298,7 @@ impl AgentService {
                 .destroy()
                 .await?;
 
-            remove_container_resources(&mut sandbox)?;
+            remove_container_resources(&mut sandbox, &cid)?;
 
             return Ok(());
         }
@@ -338,8 +330,7 @@ impl AgentService {
 
         let s = self.sandbox.clone();
         let mut sandbox = s.lock().await;
-
-        remove_container_resources(&mut sandbox)?;
+        remove_container_resources(&mut sandbox, &cid)?;
 
         Ok(())
     }
@@ -357,7 +348,7 @@ impl AgentService {
         let mut process = req
             .process
             .into_option()
-            .ok_or_else(|| anyhow!(nix::Error::EINVAL))?;
+            .ok_or_else(|| anyhow!("Unable to parse process from ExecProcessRequest"))?;
 
         // Apply any necessary corrections for PCI addresses
         update_env_pci(&mut process.Env, &sandbox.pcimap)?;
@@ -399,8 +390,22 @@ impl AgentService {
             if p.init && sig == libc::SIGTERM && !is_signal_handled(&proc_status_file, sig as u32) {
                 sig = libc::SIGKILL;
             }
-            p.signal(sig)?;
-        }
+
+            match p.signal(sig) {
+                Err(Errno::ESRCH) => {
+                    info!(
+                        sl!(),
+                        "signal encounter ESRCH, continue";
+                        "container-id" => cid.clone(),
+                        "exec-id" => eid.clone(),
+                        "pid" => p.pid,
+                        "signal" => sig,
+                    );
+                }
+                Err(err) => return Err(anyhow!(err)),
+                Ok(()) => (),
+            }
+        };
 
         if eid.is_empty() {
             // eid is empty, signal all the remaining processes in the container cgroup
@@ -606,7 +611,7 @@ impl AgentService {
         };
 
         if reader.is_none() {
-            return Err(anyhow!(nix::Error::EINVAL));
+            return Err(anyhow!("Unable to determine stream reader, is None"));
         }
 
         let reader = reader.ok_or_else(|| anyhow!("cannot get stream reader"))?;
@@ -1728,6 +1733,35 @@ fn update_container_namespaces(
     Ok(())
 }
 
+fn remove_container_resources(sandbox: &mut Sandbox, cid: &str) -> Result<()> {
+    let mut cmounts: Vec<String> = vec![];
+
+    // Find the sandbox storage used by this container
+    let mounts = sandbox.container_mounts.get(cid);
+    if let Some(mounts) = mounts {
+        for m in mounts.iter() {
+            if sandbox.storages.get(m).is_some() {
+                cmounts.push(m.to_string());
+            }
+        }
+    }
+
+    for m in cmounts.iter() {
+        if let Err(err) = sandbox.unset_and_remove_sandbox_storage(m) {
+            error!(
+                sl!(),
+                "failed to unset_and_remove_sandbox_storage for container {}, error: {:?}",
+                cid,
+                err
+            );
+        }
+    }
+
+    sandbox.container_mounts.remove(cid);
+    sandbox.containers.remove(cid);
+    Ok(())
+}
+
 fn append_guest_hooks(s: &Sandbox, oci: &mut Spec) -> Result<()> {
     if let Some(ref guest_hooks) = s.hooks {
         let mut hooks = oci.hooks.take().unwrap_or_default();
@@ -1819,7 +1853,11 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
     let path = PathBuf::from(req.path.as_str());
 
     if !path.starts_with(CONTAINER_BASE) {
-        return Err(anyhow!(nix::Error::EINVAL));
+        return Err(anyhow!(
+            "Path {:?} does not start with {}",
+            path,
+            CONTAINER_BASE
+        ));
     }
 
     let parent = path.parent();
@@ -1987,15 +2025,18 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{
-        assert_result, namespace::Namespace, protocols::agent_ttrpc_async::AgentService as _,
-        skip_if_not_root,
-    };
+    use crate::{namespace::Namespace, protocols::agent_ttrpc_async::AgentService as _};
     use nix::mount;
     use nix::sched::{unshare, CloneFlags};
     use oci::{Hook, Hooks, Linux, LinuxNamespace};
     use tempfile::{tempdir, TempDir};
+    use test_utils::{assert_result, skip_if_not_root};
     use ttrpc::{r#async::TtrpcContext, MessageHeader};
+    use which::which;
+
+    fn check_command(cmd: &str) -> bool {
+        which(cmd).is_ok()
+    }
 
     fn mk_ttrpc_context() -> TtrpcContext {
         TtrpcContext {
@@ -2060,6 +2101,7 @@ mod tests {
         let result = load_kernel_module(&m);
         assert!(result.is_err(), "load module should failed");
 
+        skip_if_not_root!();
         // case 3: normal module.
         // normally this module should eixsts...
         m.name = "bridge".to_string();
@@ -2714,6 +2756,18 @@ OtherField:other
     async fn test_ip_tables() {
         skip_if_not_root!();
 
+        if !check_command(IPTABLES_SAVE)
+            || !check_command(IPTABLES_RESTORE)
+            || !check_command(IP6TABLES_SAVE)
+            || !check_command(IP6TABLES_RESTORE)
+        {
+            warn!(
+                sl!(),
+                "one or more commands for ip tables test are missing, skip it"
+            );
+            return;
+        }
+
         let logger = slog::Logger::root(slog::Discard, o!());
         let sandbox = Sandbox::new(&logger).unwrap();
         let agent_service = Box::new(AgentService {

src/agent/src/sandbox.rs

@@ -471,7 +471,7 @@ fn online_memory(logger: &Logger) -> Result<()> {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::{mount::baremount, skip_if_not_root};
+    use crate::mount::baremount;
     use anyhow::{anyhow, Error};
     use nix::mount::MsFlags;
     use oci::{Linux, Root, Spec};
@@ -484,6 +484,7 @@ mod tests {
     use std::os::unix::fs::PermissionsExt;
     use std::path::Path;
     use tempfile::{tempdir, Builder, TempDir};
+    use test_utils::skip_if_not_root;
 
     fn bind_mount(src: &str, dst: &str, logger: &Logger) -> Result<(), Error> {
         let src_path = Path::new(src);

src/agent/src/test_utils.rs

@@ -1,99 +0,0 @@
-// Copyright (c) 2019 Intel Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-#![allow(clippy::module_inception)]
-
-#[cfg(test)]
-pub mod test_utils {
-    #[derive(Debug, PartialEq)]
-    pub enum TestUserType {
-        RootOnly,
-        NonRootOnly,
-        Any,
-    }
-
-    #[macro_export]
-    macro_rules! skip_if_root {
-        () => {
-            if nix::unistd::Uid::effective().is_root() {
-                println!("INFO: skipping {} which needs non-root", module_path!());
-                return;
-            }
-        };
-    }
-
-    #[macro_export]
-    macro_rules! skip_if_not_root {
-        () => {
-            if !nix::unistd::Uid::effective().is_root() {
-                println!("INFO: skipping {} which needs root", module_path!());
-                return;
-            }
-        };
-    }
-
-    #[macro_export]
-    macro_rules! skip_loop_if_root {
-        ($msg:expr) => {
-            if nix::unistd::Uid::effective().is_root() {
-                println!(
-                    "INFO: skipping loop {} in {} which needs non-root",
-                    $msg,
-                    module_path!()
-                );
-                continue;
-            }
-        };
-    }
-
-    #[macro_export]
-    macro_rules! skip_loop_if_not_root {
-        ($msg:expr) => {
-            if !nix::unistd::Uid::effective().is_root() {
-                println!(
-                    "INFO: skipping loop {} in {} which needs root",
-                    $msg,
-                    module_path!()
-                );
-                continue;
-            }
-        };
-    }
-
-    // Parameters:
-    //
-    // 1: expected Result
-    // 2: actual Result
-    // 3: string used to identify the test on error
-    #[macro_export]
-    macro_rules! assert_result {
-        ($expected_result:expr, $actual_result:expr, $msg:expr) => {
-            if $expected_result.is_ok() {
-                let expected_value = $expected_result.as_ref().unwrap();
-                let actual_value = $actual_result.unwrap();
-                assert!(*expected_value == actual_value, "{}", $msg);
-            } else {
-                assert!($actual_result.is_err(), "{}", $msg);
-
-                let expected_error = $expected_result.as_ref().unwrap_err();
-                let expected_error_msg = format!("{:?}", expected_error);
-
-                let actual_error_msg = format!("{:?}", $actual_result.unwrap_err());
-
-                assert!(expected_error_msg == actual_error_msg, "{}", $msg);
-            }
-        };
-    }
-
-    #[macro_export]
-    macro_rules! skip_loop_by_user {
-        ($msg:expr, $user:expr) => {
-            if $user == TestUserType::RootOnly {
-                skip_loop_if_not_root!($msg);
-            } else if $user == TestUserType::NonRootOnly {
-                skip_loop_if_root!($msg);
-            }
-        };
-    }
-}

src/agent/src/watcher.rs

@@ -528,10 +528,10 @@ impl BindWatcher {
 mod tests {
     use super::*;
     use crate::mount::is_mounted;
-    use crate::skip_if_not_root;
     use nix::unistd::{Gid, Uid};
     use std::fs;
     use std::thread;
+    use test_utils::skip_if_not_root;
 
     async fn create_test_storage(dir: &Path, id: &str) -> Result<(protos::Storage, PathBuf)> {
         let src_path = dir.join(format!("src{}", id));

src/agent/vsock-exporter/Cargo.toml

@@ -3,11 +3,12 @@ name = "vsock-exporter"
 version = "0.1.0"
 authors = ["James O. D. Hunt <james.o.hunt@intel.com>"]
 edition = "2018"
+license = "Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-nix = "0.23.0"
+nix = "0.24.2"
 libc = "0.2.94"
 thiserror = "1.0.26"
 opentelemetry = { version = "0.14.0", features=["serialize"] }

src/dragonball/Cargo.toml

@@ -28,7 +28,7 @@ lazy_static = "1.2"
 libc = "0.2.39"
 linux-loader = "0.4.0"
 log = "0.4.14"
-nix = "0.23.1"
+nix = "0.24.2"
 seccompiler = "0.2.0"
 serde = "1.0.27"
 serde_derive = "1.0.27"

src/dragonball/src/config_manager.rs

@@ -757,4 +757,67 @@ mod tests {
         assert_eq!(config5.content, config3.content);
         assert_eq!(configs.len(), 0);
     }
+
+    #[test]
+    fn test_rate_limiter_configs() {
+        const SIZE: u64 = 1024 * 1024;
+        const ONE_TIME_BURST: u64 = 1024;
+        const REFILL_TIME: u64 = 1000;
+
+        let c: TokenBucketConfigInfo = TokenBucketConfigInfo {
+            size: SIZE,
+            one_time_burst: ONE_TIME_BURST,
+            refill_time: REFILL_TIME,
+        };
+        let b: TokenBucket = c.into();
+        assert_eq!(b.capacity(), SIZE);
+        assert_eq!(b.one_time_burst(), ONE_TIME_BURST);
+        assert_eq!(b.refill_time_ms(), REFILL_TIME);
+
+        let mut rlc = RateLimiterConfigInfo {
+            bandwidth: TokenBucketConfigInfo {
+                size: SIZE,
+                one_time_burst: ONE_TIME_BURST,
+                refill_time: REFILL_TIME,
+            },
+            ops: TokenBucketConfigInfo {
+                size: SIZE * 2,
+                one_time_burst: 0,
+                refill_time: REFILL_TIME * 2,
+            },
+        };
+        let rl: RateLimiter = (&rlc).try_into().unwrap();
+        assert_eq!(rl.bandwidth().unwrap().capacity(), SIZE);
+        assert_eq!(rl.bandwidth().unwrap().one_time_burst(), ONE_TIME_BURST);
+        assert_eq!(rl.bandwidth().unwrap().refill_time_ms(), REFILL_TIME);
+        assert_eq!(rl.ops().unwrap().capacity(), SIZE * 2);
+        assert_eq!(rl.ops().unwrap().one_time_burst(), 0);
+        assert_eq!(rl.ops().unwrap().refill_time_ms(), REFILL_TIME * 2);
+
+        let bandwidth = TokenBucketConfigInfo {
+            size: SIZE * 2,
+            one_time_burst: ONE_TIME_BURST * 2,
+            refill_time: REFILL_TIME * 2,
+        };
+        rlc.update_bandwidth(bandwidth);
+        assert_eq!(rlc.bandwidth.size, SIZE * 2);
+        assert_eq!(rlc.bandwidth.one_time_burst, ONE_TIME_BURST * 2);
+        assert_eq!(rlc.bandwidth.refill_time, REFILL_TIME * 2);
+        assert_eq!(rlc.ops.size, SIZE * 2);
+        assert_eq!(rlc.ops.one_time_burst, 0);
+        assert_eq!(rlc.ops.refill_time, REFILL_TIME * 2);
+
+        let ops = TokenBucketConfigInfo {
+            size: SIZE * 3,
+            one_time_burst: ONE_TIME_BURST * 3,
+            refill_time: REFILL_TIME * 3,
+        };
+        rlc.update_ops(ops);
+        assert_eq!(rlc.bandwidth.size, SIZE * 2);
+        assert_eq!(rlc.bandwidth.one_time_burst, ONE_TIME_BURST * 2);
+        assert_eq!(rlc.bandwidth.refill_time, REFILL_TIME * 2);
+        assert_eq!(rlc.ops.size, SIZE * 3);
+        assert_eq!(rlc.ops.one_time_burst, ONE_TIME_BURST * 3);
+        assert_eq!(rlc.ops.refill_time, REFILL_TIME * 3);
+    }
 }

src/libs/Cargo.lock

@@ -46,6 +46,12 @@ version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
 
+[[package]]
+name = "bumpalo"
+version = "3.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1ad822118d20d2c234f427000d5acc36eabe1e29a348c89b63dd60b13f28e5d"
+
 [[package]]
 name = "byte-unit"
 version = "3.1.4"
@@ -100,14 +106,15 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.19"
+version = "0.4.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+checksum = "6127248204b9aba09a362f6c930ef6a78f2c1b2215f8a7b398c06e1083f17af0"
 dependencies = [
- "libc",
+ "js-sys",
  "num-integer",
  "num-traits",
  "time",
+ "wasm-bindgen",
  "winapi",
 ]
 
@@ -119,9 +126,9 @@ checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101"
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.2"
+version = "0.5.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"
+checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -364,6 +371,15 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
 
+[[package]]
+name = "js-sys"
+version = "0.3.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3fac17f7123a73ca62df411b1bf727ccc805daa070338fda671c86dac1bdc27"
+dependencies = [
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "kata-sys-util"
 version = "0.1.0"
@@ -404,6 +420,7 @@ dependencies = [
  "serde_json",
  "slog",
  "slog-scope",
+ "tempfile",
  "thiserror",
  "toml",
 ]
@@ -820,9 +837,9 @@ dependencies = [
 
 [[package]]
 name = "regex"
-version = "1.5.4"
+version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461"
+checksum = "4c4eb3267174b8c6c2f654116623910a0fef09c4753f8dd83db29c48a0df988b"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -831,9 +848,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.25"
+version = "0.6.27"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
+checksum = "a3f87b73ce11b1619a3c6332f45341e0047173771e8b8b73f87bfeefb7b56244"
 
 [[package]]
 name = "remove_dir_all"
@@ -1021,6 +1038,13 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "test-utils"
+version = "0.1.0"
+dependencies = [
+ "nix 0.24.2",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.30"
@@ -1195,6 +1219,60 @@ version = "0.11.0+wasi-snapshot-preview1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c53b543413a17a202f4be280a7e5c62a1c69345f5de525ee64f8cfdbc954994"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5491a68ab4500fa6b4d726bd67408630c3dbe9c4fe7bda16d5c82a1fd8c7340a"
+dependencies = [
+ "bumpalo",
+ "lazy_static",
+ "log",
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c441e177922bc58f1e12c022624b6216378e5febc2f0533e41ba443d505b80aa"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d94ac45fcf608c1f45ef53e748d35660f168490c10b23704c7779ab8f5c3048"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a89911bd99e5f3659ec4acf9c4d93b0a90fe4a2a11f15328472058edc5261be"
+
 [[package]]
 name = "which"
 version = "4.2.5"

src/libs/Cargo.toml

@@ -1,10 +1,11 @@
 [workspace]
 members = [
-    "logging",
-    "kata-types",
     "kata-sys-util",
-    "safe-path",
-    "protocols",
+    "kata-types",
+    "logging",
     "oci",
+    "protocols",
+    "safe-path",
+    "test-utils",
 ]
 resolver = "2"

src/libs/README.md

@@ -10,3 +10,4 @@ Currently it provides following library crates:
 | [system utilities](kata-sys-util/) | Collection of facilities and helpers to access system services. |
 | [types](kata-types/) | Collection of constants and data types shared by multiple Kata Containers components. |
 | [safe-path](safe-path/) | Utilities to safely resolve filesystem paths. |
+| [test utilities](test-utils/) | Utilities to share test code. |

src/libs/kata-sys-util/Cargo.toml

@@ -18,7 +18,7 @@ common-path = "=1.0.0"
 fail = "0.5.0"
 lazy_static = "1.4.0"
 libc = "0.2.100"
-nix = "0.24.1"
+nix = "0.24.2"
 once_cell = "1.9.0"
 serde_json = "1.0.73"
 slog = "2.5.2"

src/libs/kata-sys-util/src/spec.rs

@@ -10,9 +10,9 @@ use kata_types::container::ContainerType;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
-    /// unknow container type
-    #[error("unknow container type {0}")]
-    UnknowContainerType(String),
+    /// unknown container type
+    #[error("unknown container type {0}")]
+    UnknownContainerType(String),
     /// missing sandboxID
     #[error("missing sandboxID")]
     MissingSandboxID,
@@ -56,7 +56,7 @@ pub fn get_contaier_type(spec: &oci::Spec) -> Result<ContainerType, Error> {
                 "sandbox" => return Ok(ContainerType::PodSandbox),
                 "podsandbox" => return Ok(ContainerType::PodSandbox),
                 "container" => return Ok(ContainerType::PodContainer),
-                _ => return Err(Error::UnknowContainerType(type_value.clone())),
+                _ => return Err(Error::UnknownContainerType(type_value.clone())),
             }
         }
     }

src/libs/kata-types/Cargo.toml

@@ -15,7 +15,7 @@ byte-unit = "3.1.4"
 glob = "0.3.0"
 lazy_static = "1.4.0"
 num_cpus = "1.13.1"
-regex = "1.5.4"
+regex = "1.5.6"
 serde = { version = "1.0.100", features = ["derive"] }
 slog = "2.5.2"
 slog-scope = "4.4.0"
@@ -26,6 +26,8 @@ toml = "0.5.8"
 oci = { path = "../oci" }
 
 [dev-dependencies]
+tempfile = "3"
+
 [features]
 default = []
 enable-vendor = []

src/libs/kata-types/src/annotations/cri_containerd.rs

@@ -11,3 +11,14 @@ pub const SANDBOX: &str = "sandbox";
 pub const CONTAINER: &str = "container";
 
 pub const SANDBOX_ID_LABEL_KEY: &str = "io.kubernetes.cri.sandbox-id";
+
+// Ref: https://pkg.go.dev/github.com/containerd/containerd@v1.6.7/pkg/cri/annotations
+// SandboxCPU annotations are based on the initial CPU configuration for the sandbox. This is calculated as the
+// sum of container CPU resources, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig
+pub const SANDBOX_CPU_QUOTA_KEY: &str = "io.kubernetes.cri.sandbox-cpu-quota";
+pub const SANDBOX_CPU_PERIOD_KEY: &str = "io.kubernetes.cri.sandbox-cpu-period";
+pub const SANDBOX_CPU_SHARE_KEY: &str = "io.kubernetes.cri.sandbox-cpu-shares";
+
+// SandboxMemory is the initial amount of memory associated with this sandbox. This is calculated as the sum
+// of container memory, optionally provided by Kubelet (introduced in 1.23) as part of the PodSandboxConfig
+pub const SANDBOX_MEM_KEY: &str = "io.kubernetes.cri.sandbox-memory";

src/libs/kata-types/src/annotations/mod.rs

@@ -12,10 +12,16 @@ use std::u32;
 
 use serde::Deserialize;
 
+use crate::config::default::DEFAULT_AGENT_TYPE_NAME;
+use crate::config::default::DEFAULT_HYPERVISOR;
+use crate::config::default::DEFAULT_RUNTIME_NAME;
 use crate::config::hypervisor::get_hypervisor_plugin;
+
 use crate::config::TomlConfig;
 use crate::sl;
 
+use self::cri_containerd::{SANDBOX_CPU_PERIOD_KEY, SANDBOX_CPU_QUOTA_KEY, SANDBOX_MEM_KEY};
+
 /// CRI-containerd specific annotations.
 pub mod cri_containerd;
 
@@ -175,7 +181,7 @@ pub const KATA_ANNO_CFG_HYPERVISOR_FIRMWARE_HASH: &str =
 /// A sandbox annotation to specify cpu specific features.
 pub const KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES: &str =
     "io.katacontainers.config.hypervisor.cpu_features";
-/// A sandbox annotation for passing the default vcpus assigned for a VM by the hypervisor.
+/// A sandbox annotation for passing the default vCPUs assigned for a VM by the hypervisor.
 pub const KATA_ANNO_CFG_HYPERVISOR_DEFAULT_VCPUS: &str =
     "io.katacontainers.config.hypervisor.default_vcpus";
 /// A sandbox annotation that specifies the maximum number of vCPUs allocated for the VM by the hypervisor.
@@ -196,7 +202,7 @@ pub const KATA_ANNO_CFG_HYPERVISOR_IOMMU: &str = "io.katacontainers.config.hyper
 pub const KATA_ANNO_CFG_HYPERVISOR_IOMMU_PLATFORM: &str =
     "io.katacontainers.config.hypervisor.enable_iommu_platform";
 
-//	Hypervisor Machine related annotations
+// Hypervisor Machine related annotations
 /// A sandbox annotation to specify the type of machine being emulated by the hypervisor.
 pub const KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE: &str =
     "io.katacontainers.config.hypervisor.machine_type";
@@ -254,7 +260,7 @@ pub const KATA_ANNO_CFG_HYPERVISOR_ENABLE_ROOTLESS_HYPERVISOR: &str =
     "io.katacontainers.config.hypervisor.rootless";
 
 // Hypervisor Shared File System related annotations
-/// A sandbox annotation to specify the shared file system type, either virtio-9p or virtio-fs.
+/// A sandbox annotation to specify the shared file system type, either inline-virtio-fs (default), virtio-9p, virtio-fs or virtio-fs-nydus.
 pub const KATA_ANNO_CFG_HYPERVISOR_SHARED_FS: &str =
     "io.katacontainers.config.hypervisor.shared_fs";
 /// A sandbox annotations to specify virtio-fs vhost-user daemon path.
@@ -376,6 +382,28 @@ impl Annotation {
         self.get(CONTAINER_TYPE_KEY)
     }
 
+    /// Get the annotation of cpu quota for sandbox
+    pub fn get_sandbox_cpu_quota(&self) -> i64 {
+        let value = self
+            .get_value::<i64>(SANDBOX_CPU_QUOTA_KEY)
+            .unwrap_or(Some(0));
+        value.unwrap_or(0)
+    }
+
+    /// Get the annotation of cpu period for sandbox
+    pub fn get_sandbox_cpu_period(&self) -> u64 {
+        let value = self
+            .get_value::<u64>(SANDBOX_CPU_PERIOD_KEY)
+            .unwrap_or(Some(0));
+        value.unwrap_or(0)
+    }
+
+    /// Get the annotation of memory for sandbox
+    pub fn get_sandbox_mem(&self) -> i64 {
+        let value = self.get_value::<i64>(SANDBOX_MEM_KEY).unwrap_or(Some(0));
+        value.unwrap_or(0)
+    }
+
     /// Get the annotation to specify the Resources.Memory.Swappiness.
     pub fn get_container_resource_swappiness(&self) -> Result<Option<u32>> {
         match self.get_value::<u32>(KATA_ANNO_CONTAINER_RES_SWAPPINESS) {
@@ -415,6 +443,18 @@ impl Annotation {
                 config.runtime.agent_name = ag.to_string();
             }
         }
+
+        // set default values for runtime.name, runtime.hypervisor_name and runtime.agent
+        if config.runtime.name.is_empty() {
+            config.runtime.name = DEFAULT_RUNTIME_NAME.to_string()
+        }
+        if config.runtime.hypervisor_name.is_empty() {
+            config.runtime.hypervisor_name = DEFAULT_HYPERVISOR.to_string()
+        }
+        if config.runtime.agent_name.is_empty() {
+            config.runtime.agent_name = DEFAULT_AGENT_TYPE_NAME.to_string()
+        }
+
         let hypervisor_name = &config.runtime.hypervisor_name;
         let agent_name = &config.runtime.agent_name;
 
@@ -428,7 +468,7 @@ impl Annotation {
             if hv.security_info.is_annotation_enabled(key) {
                 match key.as_str() {
                     // update hypervisor config
-                    //	Hypervisor related annotations
+                    // Hypervisor related annotations
                     KATA_ANNO_CFG_HYPERVISOR_PATH => {
                         hv.validate_hypervisor_path(value)?;
                         hv.path = value.to_string();
@@ -451,7 +491,7 @@ impl Annotation {
                             return Err(bool_err);
                         }
                     },
-                    //	Hypervisor Block Device related annotations
+                    // Hypervisor Block Device related annotations
                     KATA_ANNO_CFG_HYPERVISOR_BLOCK_DEV_DRIVER => {
                         hv.blockdev_info.block_device_driver = value.to_string();
                     }
@@ -547,7 +587,7 @@ impl Annotation {
                         hv.boot_info.validate_boot_path(value)?;
                         hv.boot_info.firmware = value.to_string();
                     }
-                    //	Hypervisor CPU related annotations
+                    // Hypervisor CPU related annotations
                     KATA_ANNO_CFG_HYPERVISOR_CPU_FEATURES => {
                         hv.cpu_info.cpu_features = value.to_string();
                     }
@@ -587,7 +627,7 @@ impl Annotation {
                             }
                         }
                     }
-                    //	Hypervisor Device related annotations
+                    // Hypervisor Device related annotations
                     KATA_ANNO_CFG_HYPERVISOR_HOTPLUG_VFIO_ON_ROOT_BUS => {
                         match self.get_value::<bool>(key) {
                             Ok(r) => {
@@ -622,7 +662,7 @@ impl Annotation {
                             return Err(bool_err);
                         }
                     },
-                    //	Hypervisor Machine related annotations
+                    // Hypervisor Machine related annotations
                     KATA_ANNO_CFG_HYPERVISOR_MACHINE_TYPE => {
                         hv.machine_info.machine_type = value.to_string();
                     }
@@ -633,7 +673,7 @@ impl Annotation {
                         hv.machine_info.validate_entropy_source(value)?;
                         hv.machine_info.entropy_source = value.to_string();
                     }
-                    //	Hypervisor Memory related annotations
+                    // Hypervisor Memory related annotations
                     KATA_ANNO_CFG_HYPERVISOR_DEFAULT_MEMORY => {
                         match byte_unit::Byte::from_str(value) {
                             Ok(mem_bytes) => {
@@ -722,7 +762,7 @@ impl Annotation {
                             return Err(bool_err);
                         }
                     },
-                    //	Hypervisor Network related annotations
+                    // Hypervisor Network related annotations
                     KATA_ANNO_CFG_HYPERVISOR_DISABLE_VHOST_NET => match self.get_value::<bool>(key)
                     {
                         Ok(r) => {
@@ -752,7 +792,7 @@ impl Annotation {
                             }
                         }
                     }
-                    //	Hypervisor Security related annotations
+                    // Hypervisor Security related annotations
                     KATA_ANNO_CFG_HYPERVISOR_GUEST_HOOK_PATH => {
                         hv.security_info.validate_path(value)?;
                         hv.security_info.guest_hook_path = value.to_string();
@@ -767,7 +807,7 @@ impl Annotation {
                             }
                         }
                     }
-                    //	Hypervisor Shared File System related annotations
+                    // Hypervisor Shared File System related annotations
                     KATA_ANNO_CFG_HYPERVISOR_SHARED_FS => {
                         hv.shared_fs.shared_fs = self.get(key);
                     }
@@ -815,7 +855,7 @@ impl Annotation {
                 }
             } else {
                 match key.as_str() {
-                    //update agent config
+                    // update agent config
                     KATA_ANNO_CFG_KERNEL_MODULES => {
                         let kernel_mod: Vec<String> =
                             value.to_string().split(';').map(str::to_string).collect();
@@ -839,7 +879,7 @@ impl Annotation {
                             return Err(u32_err);
                         }
                     },
-                    //update runtime config
+                    // update runtime config
                     KATA_ANNO_CFG_RUNTIME_NAME => {
                         let runtime = vec!["virt-container", "linux-container", "wasm-container"];
                         if runtime.contains(&value.as_str()) {

src/libs/kata-types/src/config/agent.rs

@@ -9,8 +9,13 @@ use crate::config::{ConfigOps, TomlConfig};
 
 pub use vendor::AgentVendor;
 
+use super::default::{DEFAULT_AGENT_LOG_PORT, DEFAULT_AGENT_VSOCK_PORT};
+
+/// agent name of Kata agent.
+pub const AGENT_NAME_KATA: &str = "kata";
+
 /// Kata agent configuration information.
-#[derive(Debug, Default, Deserialize, Serialize)]
+#[derive(Debug, Default, Deserialize, Serialize, Clone)]
 pub struct Agent {
     /// If enabled, the agent will log additional debug messages to the system log.
     #[serde(default, rename = "enable_debug")]
@@ -34,11 +39,11 @@ pub struct Agent {
     pub debug_console_enabled: bool,
 
     /// Agent server port
-    #[serde(default)]
+    #[serde(default = "default_server_port")]
     pub server_port: u32,
 
     /// Agent log port
-    #[serde(default)]
+    #[serde(default = "default_log_port")]
     pub log_port: u32,
 
     /// Agent connection dialing timeout value in millisecond
@@ -75,23 +80,31 @@ pub struct Agent {
     pub container_pipe_size: u32,
 }
 
+fn default_server_port() -> u32 {
+    DEFAULT_AGENT_VSOCK_PORT
+}
+
+fn default_log_port() -> u32 {
+    DEFAULT_AGENT_LOG_PORT
+}
+
 fn default_dial_timeout() -> u32 {
-    // 10ms
+    // ms
     10
 }
 
 fn default_reconnect_timeout() -> u32 {
-    // 3s
+    // ms
     3_000
 }
 
 fn default_request_timeout() -> u32 {
-    // 30s
+    // ms
     30_000
 }
 
 fn default_health_check_timeout() -> u32 {
-    // 90s
+    // ms
     90_000
 }
 

src/libs/kata-types/src/config/default.rs

@@ -6,6 +6,9 @@
 //! Default configuration values.
 #![allow(missing_docs)]
 
+use crate::config::agent::AGENT_NAME_KATA;
+use crate::config::hypervisor::HYPERVISOR_NAME_DRAGONBALL;
+use crate::config::runtime::RUNTIME_NAME_VIRTCONTAINER;
 use lazy_static::lazy_static;
 
 lazy_static! {
@@ -16,6 +19,12 @@ lazy_static! {
     ];
 }
 pub const DEFAULT_AGENT_NAME: &str = "kata-agent";
+pub const DEFAULT_AGENT_VSOCK_PORT: u32 = 1024;
+pub const DEFAULT_AGENT_LOG_PORT: u32 = 1025;
+pub const DEFAULT_AGENT_TYPE_NAME: &str = AGENT_NAME_KATA;
+
+pub const DEFAULT_RUNTIME_NAME: &str = RUNTIME_NAME_VIRTCONTAINER;
+pub const DEFAULT_HYPERVISOR: &str = HYPERVISOR_NAME_DRAGONBALL;
 
 pub const DEFAULT_INTERNETWORKING_MODEL: &str = "tcfilter";
 

src/libs/kata-types/src/config/drop_in.rs

@@ -0,0 +1,384 @@
+// Copyright Red Hat
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+pub use drop_in_directory_handling::load;
+
+mod toml_tree_ops {
+    // The following pair of functions implement toml::Value tree merging, with
+    // the second argument being merged into the first one and consumed in the
+    // process.  The toml parser crate in use here doesn't support parsing into
+    // a pre-existing (possibly pre-filled) TomlConfig instance but can parse
+    // into a toml::Value tree so we use that instead.  All files (base and
+    // drop-ins) are initially parsed into toml::Value trees which are
+    // subsequently merged.  Only when the fully merged tree is computed it is
+    // converted to a TomlConfig instance.
+
+    fn merge_tables(base_table: &mut toml::value::Table, dropin_table: toml::value::Table) {
+        for (key, val) in dropin_table.into_iter() {
+            match base_table.get_mut(&key) {
+                Some(base_val) => merge(base_val, val),
+                None => {
+                    base_table.insert(key, val);
+                }
+            }
+        }
+    }
+
+    pub fn merge(base: &mut toml::Value, dropin: toml::Value) {
+        match dropin {
+            toml::Value::Table(dropin_table) => {
+                if let toml::Value::Table(base_table) = base {
+                    merge_tables(base_table, dropin_table);
+                } else {
+                    *base = toml::Value::Table(dropin_table);
+                }
+            }
+
+            _ => *base = dropin,
+        }
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use super::*;
+
+        // Mock config structure to stand in for TomlConfig for low-level
+        // toml::Value trees merging.
+        #[derive(Deserialize, Debug, Default, PartialEq)]
+        struct SubConfig {
+            #[serde(default)]
+            another_string: String,
+            #[serde(default)]
+            yet_another_number: i32,
+            #[serde(default)]
+            sub_array: Vec<i32>,
+        }
+
+        #[derive(Deserialize, Debug, Default, PartialEq)]
+        struct Config {
+            #[serde(default)]
+            number: i32,
+            #[serde(default)]
+            string: String,
+            #[serde(default)]
+            another_number: u8,
+            #[serde(default)]
+            array: Vec<i32>,
+
+            #[serde(default)]
+            sub: SubConfig,
+        }
+
+        #[test]
+        fn dropin_does_not_interfere_with_base() {
+            let mut base: toml::Value = toml::from_str(
+                r#"
+                number = 42
+            "#,
+            )
+            .unwrap();
+
+            let dropin: toml::Value = toml::from_str(
+                r#"
+                string = "foo"
+            "#,
+            )
+            .unwrap();
+
+            merge(&mut base, dropin);
+
+            assert_eq!(
+                base.try_into(),
+                Ok(Config {
+                    number: 42,
+                    string: "foo".into(),
+                    sub: Default::default(),
+                    ..Default::default()
+                })
+            );
+        }
+
+        #[test]
+        fn dropin_overrides_base() {
+            let mut base: toml::Value = toml::from_str(
+                r#"
+                number = 42
+                [sub]
+                another_string = "foo"
+            "#,
+            )
+            .unwrap();
+
+            let dropin: toml::Value = toml::from_str(
+                r#"
+                number = 43
+                [sub]
+                another_string = "bar"
+            "#,
+            )
+            .unwrap();
+
+            merge(&mut base, dropin);
+
+            assert_eq!(
+                base.try_into(),
+                Ok(Config {
+                    number: 43,
+                    sub: SubConfig {
+                        another_string: "bar".into(),
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                })
+            );
+        }
+
+        #[test]
+        fn dropin_extends_base() {
+            let mut base: toml::Value = toml::from_str(
+                r#"
+                number = 42
+                [sub]
+                another_string = "foo"
+            "#,
+            )
+            .unwrap();
+
+            let dropin: toml::Value = toml::from_str(
+                r#"
+                string = "hello"
+                [sub]
+                yet_another_number = 13
+            "#,
+            )
+            .unwrap();
+
+            merge(&mut base, dropin);
+
+            assert_eq!(
+                base.try_into(),
+                Ok(Config {
+                    number: 42,
+                    string: "hello".into(),
+                    sub: SubConfig {
+                        another_string: "foo".into(),
+                        yet_another_number: 13,
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                })
+            );
+        }
+
+        // Drop-ins can change the type of a value.  This might look weird but at
+        // this level we have no idea about semantics so we just do what the
+        // .toml's tell us.  The final type check is only performed by try_into().
+        // Also, we don't necessarily test this because it's a desired feature.
+        // It's just something that seems to follow from the way Value tree
+        // merging is implemented so why not acknowledge and verify it.
+        #[test]
+        fn dropin_overrides_base_type() {
+            let mut base: toml::Value = toml::from_str(
+                r#"
+                number = "foo"
+                [sub]
+                another_string = 42
+            "#,
+            )
+            .unwrap();
+
+            let dropin: toml::Value = toml::from_str(
+                r#"
+                number = 42
+                [sub]
+                another_string = "foo"
+            "#,
+            )
+            .unwrap();
+
+            merge(&mut base, dropin);
+
+            assert_eq!(
+                base.try_into(),
+                Ok(Config {
+                    number: 42,
+                    sub: SubConfig {
+                        another_string: "foo".into(),
+                        ..Default::default()
+                    },
+                    ..Default::default()
+                })
+            );
+        }
+    }
+}
+
+mod drop_in_directory_handling {
+    use crate::config::TomlConfig;
+    use std::fs;
+    use std::io::{self, Result};
+    use std::path::{Path, PathBuf};
+
+    fn get_dropin_dir_path(base_cfg_file_path: &Path) -> Result<PathBuf> {
+        let mut dropin_dir = base_cfg_file_path.to_path_buf();
+        if !dropin_dir.pop() {
+            return Err(io::Error::new(
+                io::ErrorKind::InvalidInput,
+                "base cfg file path too short",
+            ));
+        }
+        dropin_dir.push("config.d");
+        Ok(dropin_dir)
+    }
+
+    fn update_from_dropin(base_config: &mut toml::Value, dropin_file: &fs::DirEntry) -> Result<()> {
+        if !dropin_file.file_type()?.is_file() {
+            return Err(io::Error::new(
+                io::ErrorKind::Other,
+                "drop-in cfg file can only be a regular file or a symlink",
+            ));
+        }
+        let dropin_contents = fs::read_to_string(&dropin_file.path())?;
+        let dropin_config: toml::Value = toml::from_str(&dropin_contents)?;
+        super::toml_tree_ops::merge(base_config, dropin_config);
+        Ok(())
+    }
+
+    fn update_from_dropins(base_config: &mut toml::Value, dropin_dir: &Path) -> Result<()> {
+        let dropin_files_iter = match fs::read_dir(dropin_dir) {
+            Ok(iter) => iter,
+            Err(err) => {
+                if err.kind() == io::ErrorKind::NotFound {
+                    return Ok(());
+                } else {
+                    return Err(err);
+                }
+            }
+        };
+
+        let mut dropin_files = dropin_files_iter.collect::<Result<Vec<_>>>()?;
+        dropin_files.sort_by_key(|direntry| direntry.file_name());
+        for dropin_file in &dropin_files {
+            update_from_dropin(base_config, dropin_file)?;
+        }
+        Ok(())
+    }
+
+    pub fn load(base_cfg_file_path: &Path) -> Result<TomlConfig> {
+        let base_toml_str = fs::read_to_string(&base_cfg_file_path)?;
+        let mut base_config: toml::Value = toml::from_str(&base_toml_str)?;
+        let dropin_dir = get_dropin_dir_path(base_cfg_file_path)?;
+
+        update_from_dropins(&mut base_config, &dropin_dir)?;
+
+        let config: TomlConfig = base_config.try_into()?;
+        Ok(config)
+    }
+
+    #[cfg(test)]
+    mod tests {
+        use super::*;
+        use std::io::Write;
+
+        const BASE_CONFIG_DATA: &str = r#"
+            [hypervisor.qemu]
+            path = "/usr/bin/qemu-kvm"
+            default_bridges = 3
+            [runtime]
+            enable_debug = true
+            internetworking_model="tcfilter"
+        "#;
+
+        fn check_base_config(config: &TomlConfig) {
+            assert_eq!(
+                config.hypervisor["qemu"].path,
+                "/usr/bin/qemu-kvm".to_string()
+            );
+            assert_eq!(config.hypervisor["qemu"].device_info.default_bridges, 3);
+            assert!(config.runtime.debug);
+            assert_eq!(config.runtime.internetworking_model, "tcfilter".to_string());
+        }
+
+        fn create_file(path: &Path, contents: &[u8]) -> Result<()> {
+            fs::File::create(path)?.write_all(contents)
+        }
+
+        #[test]
+        fn test_no_dropins_dir() {
+            let tmpdir = tempfile::tempdir().unwrap();
+
+            let config_path = tmpdir.path().join("runtime.toml");
+            create_file(&config_path, BASE_CONFIG_DATA.as_bytes()).unwrap();
+
+            let config = load(&config_path).unwrap();
+            check_base_config(&config);
+        }
+
+        #[test]
+        fn test_no_dropins() {
+            let tmpdir = tempfile::tempdir().unwrap();
+
+            let config_path = tmpdir.path().join("runtime.toml");
+            create_file(&config_path, BASE_CONFIG_DATA.as_bytes()).unwrap();
+
+            let dropin_dir = tmpdir.path().join("config.d");
+            fs::create_dir(&dropin_dir).unwrap();
+
+            let config = load(&config_path).unwrap();
+            check_base_config(&config);
+        }
+
+        #[test]
+        fn test_dropins() {
+            let tmpdir = tempfile::tempdir().unwrap();
+
+            let dropin_data = r#"
+                [hypervisor.qemu]
+                default_vcpus = 2
+                default_bridges = 4
+                shared_fs = "virtio-fs"
+                [runtime]
+                sandbox_cgroup_only=true
+                internetworking_model="macvtap"
+                vfio_mode="guest-kernel"
+            "#;
+
+            let dropin_override_data = r#"
+                [hypervisor.qemu]
+                shared_fs = "virtio-9p"
+                [runtime]
+                vfio_mode="vfio"
+            "#;
+
+            let config_path = tmpdir.path().join("runtime.toml");
+            create_file(&config_path, BASE_CONFIG_DATA.as_bytes()).unwrap();
+
+            let dropin_dir = tmpdir.path().join("config.d");
+            fs::create_dir(&dropin_dir).unwrap();
+
+            let dropin_path = dropin_dir.join("10-base");
+            create_file(&dropin_path, dropin_data.as_bytes()).unwrap();
+
+            let dropin_override_path = dropin_dir.join("20-override");
+            create_file(&dropin_override_path, dropin_override_data.as_bytes()).unwrap();
+
+            let config = load(&config_path).unwrap();
+            assert_eq!(
+                config.hypervisor["qemu"].path,
+                "/usr/bin/qemu-kvm".to_string()
+            );
+            assert_eq!(config.hypervisor["qemu"].cpu_info.default_vcpus, 2);
+            assert_eq!(config.hypervisor["qemu"].device_info.default_bridges, 4);
+            assert_eq!(
+                config.hypervisor["qemu"].shared_fs.shared_fs.as_deref(),
+                Some("virtio-9p")
+            );
+            assert!(config.runtime.debug);
+            assert!(config.runtime.sandbox_cgroup_only);
+            assert_eq!(config.runtime.internetworking_model, "macvtap".to_string());
+            assert_eq!(config.runtime.vfio_mode, "vfio".to_string());
+        }
+    }
+}

src/libs/kata-types/src/config/hypervisor/mod.rs

@@ -32,7 +32,7 @@ use regex::RegexSet;
 
 use super::{default, ConfigOps, ConfigPlugin, TomlConfig};
 use crate::annotations::KATA_ANNO_CFG_HYPERVISOR_PREFIX;
-use crate::{eother, resolve_path, validate_path};
+use crate::{eother, resolve_path, sl, validate_path};
 
 mod dragonball;
 pub use self::dragonball::{DragonballConfig, HYPERVISOR_NAME_DRAGONBALL};
@@ -320,7 +320,7 @@ impl CpuInfo {
     }
 }
 
-/// Configuration information for shared filesystem, such virtio-9p and virtio-fs.
+/// Configuration information for debug
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct DebugInfo {
     /// This option changes the default hypervisor and kernel parameters to enable debug output
@@ -596,7 +596,7 @@ impl MemoryInfo {
     }
 }
 
-/// Configuration information for virtual machine.
+/// Configuration information for network.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct NetworkInfo {
     /// If vhost-net backend for virtio-net is not desired, set to true.
@@ -638,7 +638,7 @@ impl NetworkInfo {
     }
 }
 
-/// Configuration information for virtual machine.
+/// Configuration information for security.
 #[derive(Clone, Debug, Default, Deserialize, Serialize)]
 pub struct SecurityInfo {
     /// Enable running QEMU VMM as a non-root user.
@@ -818,14 +818,22 @@ impl SharedFsInfo {
         validate_path_pattern(&self.valid_virtio_fs_daemon_paths, path)
     }
 
-    fn adjust_virtio_fs(&mut self, _inline: bool) -> Result<()> {
-        resolve_path!(
-            self.virtio_fs_daemon,
-            "Virtio-fs daemon path {} is invalid: {}"
-        )?;
+    fn adjust_virtio_fs(&mut self, inline: bool) -> Result<()> {
+        // inline mode doesn't need external virtiofsd daemon
+        if !inline {
+            resolve_path!(
+                self.virtio_fs_daemon,
+                "Virtio-fs daemon path {} is invalid: {}"
+            )?;
+        }
+
         if self.virtio_fs_cache.is_empty() {
             self.virtio_fs_cache = default::DEFAULT_VIRTIO_FS_CACHE_MODE.to_string();
         }
+        if self.virtio_fs_cache == *"none" {
+            warn!(sl!(), "virtio-fs cache mode `none` is deprecated since Kata Containers 2.5.0 and will be removed in the future release, please use `never` instead. For more details please refer to https://github.com/kata-containers/kata-containers/issues/4234.");
+            self.virtio_fs_cache = default::DEFAULT_VIRTIO_FS_CACHE_MODE.to_string();
+        }
         if self.virtio_fs_is_dax && self.virtio_fs_cache_size == 0 {
             self.virtio_fs_cache_size = default::DEFAULT_VIRTIO_FS_DAX_SIZE_MB;
         }
@@ -836,16 +844,13 @@ impl SharedFsInfo {
     }
 
     fn validate_virtio_fs(&self, inline: bool) -> Result<()> {
-        if inline && !self.virtio_fs_daemon.is_empty() {
-            return Err(eother!(
-                "Executable path for inline-virtio-fs is not empty: {}",
-                &self.virtio_fs_daemon
-            ));
+        // inline mode doesn't need external virtiofsd daemon
+        if !inline {
+            validate_path!(
+                self.virtio_fs_daemon,
+                "Virtio-fs daemon path {} is invalid: {}"
+            )?;
         }
-        validate_path!(
-            self.virtio_fs_daemon,
-            "Virtio-fs daemon path {} is invalid: {}"
-        )?;
 
         let l = ["none", "auto", "always"];
 

src/libs/kata-types/src/config/mod.rs

@@ -19,6 +19,7 @@ use crate::{eother, sl};
 pub mod default;
 
 mod agent;
+mod drop_in;
 pub mod hypervisor;
 
 pub use self::agent::Agent;
@@ -28,7 +29,9 @@ pub use self::hypervisor::{
 };
 
 mod runtime;
-pub use self::runtime::{Runtime, RuntimeVendor};
+pub use self::runtime::{Runtime, RuntimeVendor, RUNTIME_NAME_VIRTCONTAINER};
+
+pub use self::agent::AGENT_NAME_KATA;
 
 /// Trait to manipulate global Kata configuration information.
 pub trait ConfigPlugin: Send + Sync {
@@ -94,21 +97,15 @@ impl TomlConfig {
     /// If `config_file` is valid, it will used, otherwise a built-in default path list will be
     /// scanned.
     pub fn load_from_file<P: AsRef<Path>>(config_file: P) -> Result<(TomlConfig, PathBuf)> {
-        let file_path = if !config_file.as_ref().as_os_str().is_empty() {
-            fs::canonicalize(config_file)?
-        } else {
-            Self::get_default_config_file()?
-        };
+        let mut result = Self::load_raw_from_file(config_file);
+        if let Ok((ref mut config, _)) = result {
+            Hypervisor::adjust_config(config)?;
+            Runtime::adjust_config(config)?;
+            Agent::adjust_config(config)?;
+            info!(sl!(), "get kata config: {:?}", config);
+        }
 
-        info!(
-            sl!(),
-            "load configuration from: {}",
-            file_path.to_string_lossy()
-        );
-        let content = fs::read_to_string(&file_path)?;
-        let config = Self::load(&content)?;
-
-        Ok((config, file_path))
+        result
     }
 
     /// Load raw Kata configuration information from configuration files.
@@ -127,13 +124,15 @@ impl TomlConfig {
             "load configuration from: {}",
             file_path.to_string_lossy()
         );
-        let content = fs::read_to_string(&file_path)?;
-        let config: TomlConfig = toml::from_str(&content)?;
+        let config = drop_in::load(&file_path)?;
 
         Ok((config, file_path))
     }
 
     /// Load Kata configuration information from string.
+    ///
+    /// This function only works with `configuration.toml` and does not handle
+    /// drop-in config file fragments in config.d/.
     pub fn load(content: &str) -> Result<TomlConfig> {
         let mut config: TomlConfig = toml::from_str(content)?;
         Hypervisor::adjust_config(&mut config)?;

src/libs/kata-types/src/config/runtime.rs

@@ -10,6 +10,9 @@ use super::default;
 use crate::config::{ConfigOps, TomlConfig};
 use crate::{eother, resolve_path, validate_path};
 
+/// Type of runtime VirtContainer.
+pub const RUNTIME_NAME_VIRTCONTAINER: &str = "virt_container";
+
 /// Kata runtime configuration information.
 #[derive(Debug, Default, Deserialize, Serialize)]
 pub struct Runtime {
@@ -99,6 +102,10 @@ pub struct Runtime {
     #[serde(default)]
     pub enable_pprof: bool,
 
+    /// If enabled, static resource management will calculate the vcpu and memory for the sandbox/container
+    #[serde(default)]
+    pub static_resource_mgmt: bool,
+
     /// Determines whether container seccomp profiles are passed to the virtual machine and
     /// applied by the kata agent. If set to true, seccomp is not applied within the guest.
     #[serde(default)]

src/libs/logging/Cargo.toml

@@ -3,6 +3,7 @@ name = "logging"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

src/libs/oci/Cargo.toml

@@ -3,6 +3,7 @@ name = "oci"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 serde = "1.0.131"

src/libs/protocols/Cargo.toml

@@ -3,6 +3,7 @@ name = "protocols"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [features]
 default = []

src/libs/test-utils/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "test-utils"
+version = "0.1.0"
+description = "A library for test Rust code"
+keywords = ["kata", "container", "test", "utils"]
+categories = ["testing"]
+authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
+repository = "https://github.com/kata-containers/kata-containers.git"
+homepage = "https://katacontainers.io/"
+readme = "README.md"
+license = "Apache-2.0"
+edition = "2018"
+
+[dependencies]
+nix = "0.24.2"

src/libs/test-utils/README.md

@@ -0,0 +1,8 @@
+Test Utilities
+====================
+
+A library to share test code for Rust.
+
+## License
+
+This code is licensed under [Apache-2.0](../../../LICENSE).

src/libs/test-utils/src/lib.rs

@@ -0,0 +1,119 @@
+// Copyright (c) 2019 Intel Corporation
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#[derive(Debug, PartialEq)]
+pub enum TestUserType {
+    RootOnly,
+    NonRootOnly,
+    Any,
+}
+
+#[macro_export]
+macro_rules! skip_if_root {
+    () => {
+        if nix::unistd::Uid::effective().is_root() {
+            println!("INFO: skipping {} which needs non-root", module_path!());
+            return;
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! skip_if_not_root {
+    () => {
+        if !nix::unistd::Uid::effective().is_root() {
+            println!("INFO: skipping {} which needs root", module_path!());
+            return;
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! skip_loop_if_root {
+    ($msg:expr) => {
+        if nix::unistd::Uid::effective().is_root() {
+            println!(
+                "INFO: skipping loop {} in {} which needs non-root",
+                $msg,
+                module_path!()
+            );
+            continue;
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! skip_loop_if_not_root {
+    ($msg:expr) => {
+        if !nix::unistd::Uid::effective().is_root() {
+            println!(
+                "INFO: skipping loop {} in {} which needs root",
+                $msg,
+                module_path!()
+            );
+            continue;
+        }
+    };
+}
+
+// Parameters:
+//
+// 1: expected Result
+// 2: actual Result
+// 3: string used to identify the test on error
+#[macro_export]
+macro_rules! assert_result {
+    ($expected_result:expr, $actual_result:expr, $msg:expr) => {
+        if $expected_result.is_ok() {
+            let expected_value = $expected_result.as_ref().unwrap();
+            let actual_value = $actual_result.unwrap();
+            assert!(*expected_value == actual_value, "{}", $msg);
+        } else {
+            assert!($actual_result.is_err(), "{}", $msg);
+
+            let expected_error = $expected_result.as_ref().unwrap_err();
+            let expected_error_msg = format!("{:?}", expected_error);
+
+            let actual_error_msg = format!("{:?}", $actual_result.unwrap_err());
+
+            assert!(expected_error_msg == actual_error_msg, "{}", $msg);
+        }
+    };
+}
+
+#[macro_export]
+macro_rules! skip_loop_by_user {
+    ($msg:expr, $user:expr) => {
+        if $user == TestUserType::RootOnly {
+            skip_loop_if_not_root!($msg);
+        } else if $user == TestUserType::NonRootOnly {
+            skip_loop_if_root!($msg);
+        }
+    };
+}
+
+#[cfg(test)]
+mod tests {
+    use super::{skip_if_not_root, skip_if_root};
+
+    #[test]
+    fn test_skip_if_not_root() {
+        skip_if_not_root!();
+        assert!(
+            nix::unistd::Uid::effective().is_root(),
+            "normal user should be skipped"
+        )
+    }
+
+    #[test]
+    fn test_skip_if_root() {
+        skip_if_root!();
+        assert!(
+            !nix::unistd::Uid::effective().is_root(),
+            "root user should be skipped"
+        )
+    }
+}

src/runtime-rs/Cargo.lock

@@ -69,6 +69,15 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "anyhow"
 version = "1.0.57"
@@ -99,6 +108,108 @@ version = "0.7.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
+[[package]]
+name = "async-channel"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14485364214912d3b19cc3435dde4df66065127f05fa0d75c712f36f12c2f28"
+dependencies = [
+ "concurrent-queue",
+ "event-listener",
+ "futures-core",
+]
+
+[[package]]
+name = "async-executor"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "871f9bb5e0a22eeb7e8cf16641feb87c9dc67032ccf8ff49e772eb9941d3a965"
+dependencies = [
+ "async-task",
+ "concurrent-queue",
+ "fastrand",
+ "futures-lite",
+ "once_cell",
+ "slab",
+]
+
+[[package]]
+name = "async-global-executor"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5262ed948da60dd8956c6c5aca4d4163593dddb7b32d73267c93dab7b2e98940"
+dependencies = [
+ "async-channel",
+ "async-executor",
+ "async-io",
+ "async-lock",
+ "blocking",
+ "futures-lite",
+ "num_cpus",
+ "once_cell",
+]
+
+[[package]]
+name = "async-io"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0ab006897723d9352f63e2b13047177c3982d8d79709d713ce7747a8f19fd1b0"
+dependencies = [
+ "autocfg",
+ "concurrent-queue",
+ "futures-lite",
+ "libc",
+ "log",
+ "once_cell",
+ "parking",
+ "polling",
+ "slab",
+ "socket2",
+ "waker-fn",
+ "winapi",
+]
+
+[[package]]
+name = "async-lock"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e97a171d191782fba31bb902b14ad94e24a68145032b7eedf871ab0bc0d077b6"
+dependencies = [
+ "event-listener",
+]
+
+[[package]]
+name = "async-std"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62565bb4402e926b29953c785397c6dc0391b7b446e45008b0049eb43cec6f5d"
+dependencies = [
+ "async-channel",
+ "async-global-executor",
+ "async-io",
+ "async-lock",
+ "crossbeam-utils",
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-lite",
+ "gloo-timers",
+ "kv-log-macro",
+ "log",
+ "memchr",
+ "once_cell",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+ "wasm-bindgen-futures",
+]
+
+[[package]]
+name = "async-task"
+version = "4.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a40729d2133846d9ed0ea60a8b9541bccddab49cd30f0715a1da672fe9a2524"
+
 [[package]]
 name = "async-trait"
 version = "0.1.56"
@@ -110,6 +221,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "atomic-waker"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a"
+
 [[package]]
 name = "autocfg"
 version = "1.1.0"
@@ -192,6 +309,20 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "blocking"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6ccb65d468978a086b69884437ded69a90faab3bbe6e67f242173ea728acccc"
+dependencies = [
+ "async-channel",
+ "async-task",
+ "atomic-waker",
+ "fastrand",
+ "futures-lite",
+ "once_cell",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.10.0"
@@ -226,6 +357,12 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
 
+[[package]]
+name = "cache-padded"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c"
+
 [[package]]
 name = "caps"
 version = "0.5.3"
@@ -272,14 +409,16 @@ dependencies = [
 
 [[package]]
 name = "chrono"
-version = "0.4.19"
+version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "670ad68c9088c2a963aaa298cb369688cf3f9465ce5e2d4ca10e6e0098a1ce73"
+checksum = "bfd4d1b31faaa3a89d7934dbded3111da0d2ef28e3ebccdb4f0179f5929d1ef1"
 dependencies = [
- "libc",
+ "iana-time-zone",
+ "js-sys",
  "num-integer",
  "num-traits",
  "time 0.1.43",
+ "wasm-bindgen",
  "winapi",
 ]
 
@@ -294,8 +433,9 @@ dependencies = [
  "kata-sys-util",
  "kata-types",
  "lazy_static",
- "nix 0.24.1",
+ "nix 0.24.2",
  "oci",
+ "persist",
  "protobuf",
  "serde_json",
  "slog",
@@ -312,6 +452,15 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2382f75942f4b3be3690fe4f86365e9c853c1587d6ee58212cebf6e2a9ccd101"
 
+[[package]]
+name = "concurrent-queue"
+version = "1.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af4780a44ab5696ea9e28294517f1fffb421a83a25af521333c838635509db9c"
+dependencies = [
+ "cache-padded",
+]
+
 [[package]]
 name = "constant_time_eq"
 version = "0.1.5"
@@ -338,6 +487,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
+
 [[package]]
 name = "cpuid-bool"
 version = "0.1.2"
@@ -383,6 +538,16 @@ dependencies = [
  "typenum",
 ]
 
+[[package]]
+name = "ctor"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f877be4f7c9f246b183111634f75baa039715e3f46ce860677d3b19a69fb229c"
+dependencies = [
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "darling"
 version = "0.13.4"
@@ -606,7 +771,7 @@ version = "0.0.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8f82432ae94d42f160b6e17389d6e1c1eee29827b99ad32d35a0a96bb98bedb5"
 dependencies = [
- "core-foundation-sys",
+ "core-foundation-sys 0.2.3",
  "libc",
 ]
 
@@ -631,7 +796,7 @@ dependencies = [
  "libc",
  "linux-loader",
  "log",
- "nix 0.23.1",
+ "nix 0.24.2",
  "seccompiler",
  "serde",
  "serde_derive",
@@ -681,6 +846,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "event-listener"
+version = "2.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0"
+
 [[package]]
 name = "event-manager"
 version = "0.2.1"
@@ -758,7 +929,7 @@ dependencies = [
  "arc-swap 1.5.0",
  "bitflags",
  "caps",
- "core-foundation-sys",
+ "core-foundation-sys 0.2.3",
  "diskarbitration-sys",
  "lazy_static",
  "libc",
@@ -824,6 +995,21 @@ version = "0.3.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc4045962a5a5e935ee2fdedaa4e08284547402885ab326734432bed5d12966b"
 
+[[package]]
+name = "futures-lite"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7694489acd39452c77daa48516b894c153f192c3578d5a839b62c58099fcbf48"
+dependencies = [
+ "fastrand",
+ "futures-core",
+ "futures-io",
+ "memchr",
+ "parking",
+ "pin-project-lite",
+ "waker-fn",
+]
+
 [[package]]
 name = "futures-macro"
 version = "0.3.21"
@@ -915,6 +1101,18 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
 
+[[package]]
+name = "gloo-timers"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fb7d06c1c8cc2a29bee7ec961009a0b2caa0793ee4900c2ffb348734ba1c8f9"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "go-flag"
 version = "0.1.0"
@@ -990,8 +1188,10 @@ dependencies = [
  "kata-types",
  "libc",
  "logging",
- "nix 0.24.1",
+ "nix 0.24.2",
+ "persist",
  "seccompiler",
+ "serde",
  "serde_json",
  "slog",
  "slog-scope",
@@ -1000,6 +1200,19 @@ dependencies = [
  "vmm-sys-util",
 ]
 
+[[package]]
+name = "iana-time-zone"
+version = "0.1.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ad2bfd338099682614d3ee3fe0cd72e0b6a41ca6a87f6a74a3bd593c91650501"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys 0.8.3",
+ "js-sys",
+ "wasm-bindgen",
+ "winapi",
+]
+
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1087,9 +1300,9 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.57"
+version = "0.3.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "671a26f820db17c2a2750743f1dd03bafd15b98c9f30c7c2628c024c05d73397"
+checksum = "258451ab10b34f8af53416d1fdab72c22e805f0c92a1136d59470ec0b11138b2"
 dependencies = [
  "wasm-bindgen",
 ]
@@ -1106,7 +1319,7 @@ dependencies = [
  "kata-types",
  "lazy_static",
  "libc",
- "nix 0.24.1",
+ "nix 0.24.2",
  "oci",
  "once_cell",
  "rand 0.7.3",
@@ -1135,6 +1348,15 @@ dependencies = [
  "toml 0.5.9",
 ]
 
+[[package]]
+name = "kv-log-macro"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f"
+dependencies = [
+ "log",
+]
+
 [[package]]
 name = "kvm-bindings"
 version = "0.5.0"
@@ -1210,6 +1432,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
 dependencies = [
  "cfg-if 1.0.0",
+ "value-bag",
 ]
 
 [[package]]
@@ -1225,9 +1448,9 @@ dependencies = [
 
 [[package]]
 name = "lz4-sys"
-version = "1.9.3"
+version = "1.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7be8908e2ed6f31c02db8a9fa962f03e36c53fbfde437363eae3306b85d7e17"
+checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900"
 dependencies = [
  "cc",
  "libc",
@@ -1356,19 +1579,6 @@ dependencies = [
  "tokio",
 ]
 
-[[package]]
-name = "nix"
-version = "0.16.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dd0eaf8df8bab402257e0a5c17a254e4cc1f72a93588a1ddfb5d356c801aa7cb"
-dependencies = [
- "bitflags",
- "cc",
- "cfg-if 0.1.10",
- "libc",
- "void",
-]
-
 [[package]]
 name = "nix"
 version = "0.23.1"
@@ -1384,9 +1594,9 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.24.1"
+version = "0.24.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f17df307904acd05aa8e32e97bb20f2a0df1728bbc2d771ae8f9a90463441e9"
+checksum = "195cdbc1741b8134346d515b3a56a1c94b0912758009cfd53f99ea0f57b065fc"
 dependencies = [
  "bitflags",
  "cfg-if 1.0.0",
@@ -1507,6 +1717,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "parking"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72"
+
 [[package]]
 name = "parking_lot"
 version = "0.11.2"
@@ -1567,6 +1783,20 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
 
+[[package]]
+name = "persist"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "async-trait",
+ "kata-sys-util",
+ "kata-types",
+ "libc",
+ "safe-path",
+ "serde",
+ "serde_json",
+]
+
 [[package]]
 name = "petgraph"
 version = "0.5.1"
@@ -1589,6 +1819,20 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "polling"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "899b00b9c8ab553c743b3e11e87c5c7d423b2a2de229ba95b24a756344748011"
+dependencies = [
+ "autocfg",
+ "cfg-if 1.0.0",
+ "libc",
+ "log",
+ "wepoll-ffi",
+ "winapi",
+]
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -1929,11 +2173,13 @@ dependencies = [
  "logging",
  "netlink-packet-route",
  "netlink-sys",
- "nix 0.24.1",
+ "nix 0.24.2",
  "oci",
+ "persist",
  "rand 0.7.3",
  "rtnetlink",
  "scopeguard",
+ "serde",
  "slog",
  "slog-scope",
  "tokio",
@@ -1959,7 +2205,7 @@ dependencies = [
  "log",
  "netlink-packet-route",
  "netlink-proto",
- "nix 0.24.1",
+ "nix 0.24.2",
  "thiserror",
  "tokio",
 ]
@@ -1975,6 +2221,7 @@ dependencies = [
  "linux_container",
  "logging",
  "oci",
+ "persist",
  "slog",
  "slog-scope",
  "tokio",
@@ -2014,6 +2261,13 @@ version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
 
+[[package]]
+name = "safe-path"
+version = "0.1.0"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -2031,18 +2285,18 @@ dependencies = [
 
 [[package]]
 name = "serde"
-version = "1.0.137"
+version = "1.0.143"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+checksum = "53e8e5d5b70924f74ff5c6d64d9a5acd91422117c60f48c4e07855238a254553"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.137"
+version = "1.0.143"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
+checksum = "d3d8e8de557aee63c26b85b947f5e59b690d0454c753f3adeb5cd7835ab88391"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2051,9 +2305,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.81"
+version = "1.0.83"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
+checksum = "38dd04e3c8279e75b31ef29dbdceebfe5ad89f4d0937213c53f7d49d01b3d5a7"
 dependencies = [
  "itoa",
  "ryu",
@@ -2113,6 +2367,7 @@ dependencies = [
  "common",
  "containerd-shim-protos",
  "logging",
+ "persist",
  "runtimes",
  "slog",
  "slog-scope",
@@ -2146,7 +2401,7 @@ dependencies = [
  "libc",
  "log",
  "logging",
- "nix 0.24.1",
+ "nix 0.24.2",
  "oci",
  "protobuf",
  "rand 0.8.5",
@@ -2617,6 +2872,16 @@ dependencies = [
  "rand 0.3.23",
 ]
 
+[[package]]
+name = "value-bag"
+version = "1.0.0-alpha.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55"
+dependencies = [
+ "ctor",
+ "version_check",
+]
+
 [[package]]
 name = "version_check"
 version = "0.9.4"
@@ -2629,6 +2894,7 @@ version = "0.1.0"
 dependencies = [
  "agent",
  "anyhow",
+ "async-std",
  "async-trait",
  "awaitgroup",
  "common",
@@ -2640,8 +2906,9 @@ dependencies = [
  "lazy_static",
  "libc",
  "logging",
- "nix 0.16.1",
+ "nix 0.24.2",
  "oci",
+ "persist",
  "protobuf",
  "resource",
  "serde",
@@ -2704,12 +2971,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "void"
-version = "1.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d"
-
 [[package]]
 name = "vsock"
 version = "0.2.6"
@@ -2720,6 +2981,12 @@ dependencies = [
  "nix 0.23.1",
 ]
 
+[[package]]
+name = "waker-fn"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca"
+
 [[package]]
 name = "wasi"
 version = "0.9.0+wasi-snapshot-preview1"
@@ -2740,9 +3007,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.80"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "27370197c907c55e3f1a9fbe26f44e937fe6451368324e009cba39e139dc08ad"
+checksum = "fc7652e3f6c4706c8d9cd54832c4a4ccb9b5336e2c3bd154d5cccfbf1c1f5f7d"
 dependencies = [
  "cfg-if 1.0.0",
  "wasm-bindgen-macro",
@@ -2750,13 +3017,13 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.80"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53e04185bfa3a779273da532f5025e33398409573f348985af9a1cbf3774d3f4"
+checksum = "662cd44805586bd52971b9586b1df85cdbbd9112e4ef4d8f41559c334dc6ac3f"
 dependencies = [
  "bumpalo",
- "lazy_static",
  "log",
+ "once_cell",
  "proc-macro2",
  "quote",
  "syn",
@@ -2764,10 +3031,22 @@ dependencies = [
 ]
 
 [[package]]
-name = "wasm-bindgen-macro"
-version = "0.2.80"
+name = "wasm-bindgen-futures"
+version = "0.4.32"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "17cae7ff784d7e83a2fe7611cfe766ecf034111b49deb850a3dc7699c08251f5"
+checksum = "fa76fb221a1f8acddf5b54ace85912606980ad661ac7a503b4570ffd3a624dad"
+dependencies = [
+ "cfg-if 1.0.0",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.82"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b260f13d3012071dfb1512849c033b1925038373aea48ced3012c09df952c602"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -2775,9 +3054,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.80"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "99ec0dc7a4756fffc231aab1b9f2f578d23cd391390ab27f952ae0c9b3ece20b"
+checksum = "5be8e654bdd9b79216c2929ab90721aa82faf65c48cdf08bdc4e7f51357b80da"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -2788,9 +3067,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.80"
+version = "0.2.82"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d554b7f530dee5964d9a9468d95c1f8b8acae4f282807e7d27d4b03099a46744"
+checksum = "6598dd0bd3c7d51095ff6531a5b23e02acdc81804e30d8f07afb77b7215a140a"
 
 [[package]]
 name = "wasm_container"
@@ -2813,6 +3092,15 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "wepoll-ffi"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d743fdedc5c64377b5fc2bc036b01c7fd642205a0d96356034ae3404d49eb7fb"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "which"
 version = "4.2.5"

src/runtime-rs/Makefile

@@ -44,29 +44,43 @@ else
 endif
 
 ifeq ($(PREFIX),)
-    PREFIX := /usr
+PREFIX := /usr
+EXEC_PREFIX := $(PREFIX)/local
+##VAR BINDIR=<path> is a directory for installing executable programs
+BINDIR := $(EXEC_PREFIX)/bin
+else
+EXEC_PREFIX := $(PREFIX)
+##VAR BINDIR=<path> is a directory for installing executable programs
+# when creating the kata-deploy image, the default installation path for go runtime is $(EXEC_PREFIX)/bin, so we put it here for multiple runtime
+BINDIR := $(EXEC_PREFIX)/runtime-rs/bin/
 endif
 
-PREFIXDEPS    := $(PREFIX)
+PREFIXDEPS := $(PREFIX)
 LIBEXECDIR := $(PREFIXDEPS)/libexec
 SHAREDIR := $(PREFIX)/share
 DEFAULTSDIR := $(SHAREDIR)/defaults
 PROJECT_DIR = $(PROJECT_TAG)
 IMAGENAME = $(PROJECT_TAG).img
 TARGET = $(PROJECT_COMPONENT)
-
+SYSCONFDIR := /etc
 CONFIG_FILE = configuration.toml
 HYPERVISOR_DB = dragonball
+HYPERVISOR_ACRN = acrn
+HYPERVISOR_FC = firecracker
+HYPERVISOR_QEMU = qemu
+HYPERVISOR_CLH = cloud-hypervisor
 
 
 DEFAULT_HYPERVISOR ?= $(HYPERVISOR_DB)
 
-HYPERVISORS := $(HYPERVISOR_DB)
+# List of hypervisors this build system can generate configuration for.
+HYPERVISORS := $(HYPERVISOR_DB) $(HYPERVISOR_ACRN) $(HYPERVISOR_FC) $(HYPERVISOR_QEMU) $(HYPERVISOR_CLH)
 
 DBVALIDHYPERVISORPATHS := []
 PKGDATADIR := $(PREFIXDEPS)/share/$(PROJECT_DIR)
 KERNELDIR := $(PKGDATADIR)
 IMAGEPATH := $(PKGDATADIR)/$(IMAGENAME)
+PKGLIBEXECDIR := $(LIBEXECDIR)/$(PROJECT_DIR)
 FIRMWAREPATH :=
 FIRMWAREVOLUMEPATH :=
 
@@ -122,7 +136,7 @@ DEFMSIZE9P := 8192
 DEFVFIOMODE := guest-kernel
 # Default cgroup model
 DEFSANDBOXCGROUPONLY ?= false
-DEFSTATICRESOURCEMGMT ?= false
+DEFSTATICRESOURCEMGMT_DB ?= false
 DEFBINDMOUNTS := []
 SED = sed
 CLI_DIR = cmd
@@ -255,7 +269,7 @@ USER_VARS += DEFMSIZE9P
 USER_VARS += DEFENTROPYSOURCE
 USER_VARS += DEFVALIDENTROPYSOURCES
 USER_VARS += DEFSANDBOXCGROUPONLY
-USER_VARS += DEFSTATICRESOURCEMGMT
+USER_VARS += DEFSTATICRESOURCEMGMT_DB
 USER_VARS += DEFBINDMOUNTS
 USER_VARS += DEFVFIOMODE
 USER_VARS += BUILDFLAGS
@@ -290,9 +304,7 @@ endif
 TARGET_PATH = target/$(TRIPLE)/$(BUILD_TYPE)/$(TARGET)
 
 ##VAR DESTDIR=<path> is a directory prepended to each installed target file
-DESTDIR :=
-##VAR BINDIR=<path> is a directory for installing executable programs
-BINDIR := /usr/local/bin
+DESTDIR ?= /
 
 GENERATED_CODE = crates/shim/src/config.rs
 
@@ -330,8 +342,12 @@ define get_toolchain_version
 $(shell printf "%s: %s\\n" "toolchain" "$(or $(shell rustup show active-toolchain 2>/dev/null), (unknown))")
 endef
 
+# Install a configuration file
+# params:
+# $1 : file to install
+# $2 : directory path where file will be installed
 define INSTALL_FILE
-	install -D -m 644 $1 $(DESTDIR)$2/$1 || exit 1;
+	install --mode 0644 -D $1 $(DESTDIR)$2/$(notdir $1);
 endef
 
 # Returns the name of the kernel file to use based on the provided KERNELTYPE.
@@ -382,20 +398,62 @@ show-header:
 	@printf "%s - version %s (commit %s)\n\n" "$(TARGET)" "$(VERSION)" "$(COMMIT_MSG)"
 
 show-summary: show-header
-	@printf "project:\n"
+	@printf "• Project:\n"
 	@printf "  name: $(PROJECT_NAME)\n"
 	@printf "  url: $(PROJECT_URL)\n"
 	@printf "  component: $(PROJECT_COMPONENT)\n"
-	@printf "target: $(TARGET)\n"
-	@printf "architecture:\n"
-	@printf "  host: $(ARCH)\n"
-	@printf "rust:\n"
+	@printf "\n"
+	@printf "• Target: $(TARGET)\n"
+	@printf "\n"
+	@printf "• Architecture: $(ARCH)\n"
+	@printf "\n"
+	@printf "• Rust:\n"
 	@printf "  %s\n" "$(call get_command_version,cargo)"
 	@printf "  %s\n" "$(call get_command_version,rustc)"
 	@printf "  %s\n" "$(call get_command_version,rustup)"
 	@printf "  %s\n" "$(call get_toolchain_version)"
 	@printf "\n"
-
+	@printf "• Hypervisors:\n"
+	@printf "\tDefault: $(DEFAULT_HYPERVISOR)\n"
+	@printf "\tKnown: $(sort $(HYPERVISORS))\n"
+	@printf "\tAvailable for this architecture: $(sort $(KNOWN_HYPERVISORS))\n"
+	@printf "\n"
+	@printf "• Summary:\n"
+	@printf "\n"
+	@printf "\tdestination install path (DESTDIR) : %s\n" $(abspath $(DESTDIR))
+	@printf "\tbinary installation path (BINDIR) : %s\n" $(abspath $(BINDIR))
+	@printf "\tbinaries to install :\n"
+	@printf \
+          "$(foreach b,$(sort $(SHIMV2)),$(shell printf "\\t - $(shell readlink -m $(DESTDIR)/$(BINDIR)/$(b))\\\n"))"
+	@printf "\tconfigs to install (CONFIGS) :\n"
+	@printf \
+	  "$(foreach c,$(sort $(CONFIGS)),$(shell printf "\\t - $(c)\\\n"))"
+	@printf "\tinstall paths (CONFIG_PATHS) :\n"
+	@printf \
+	  "$(foreach c,$(sort $(CONFIG_PATHS)),$(shell printf "\\t - $(c)\\\n"))"
+	@printf "\talternate config paths (SYSCONFIG_PATHS) : %s\n"
+	@printf \
+	  "$(foreach c,$(sort $(SYSCONFIG_PATHS)),$(shell printf "\\t - $(c)\\\n"))"
+	@printf "\tdefault install path for $(DEFAULT_HYPERVISOR) (CONFIG_PATH) : %s\n" $(abspath $(CONFIG_PATH))
+	@printf "\tdefault alternate config path (SYSCONFIG) : %s\n" $(abspath $(SYSCONFIG))
+ifneq (,$(findstring $(HYPERVISOR_QEMU),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_QEMU) hypervisor path (QEMUPATH) : %s\n" $(abspath $(QEMUPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_QEMU_VIRTIOFS),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_QEMU_VIRTIOFS) hypervisor path (QEMUVIRTIOFSPATH) : %s\n" $(abspath $(QEMUVIRTIOFSPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_CLH),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_CLH) hypervisor path (CLHPATH) : %s\n" $(abspath $(CLHPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_FC),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_FC) hypervisor path (FCPATH) : %s\n" $(abspath $(FCPATH))
+endif
+ifneq (,$(findstring $(HYPERVISOR_ACRN),$(KNOWN_HYPERVISORS)))
+	@printf "\t$(HYPERVISOR_ACRN) hypervisor path (ACRNPATH) : %s\n" $(abspath $(ACRNPATH))
+endif
+	@printf "\tassets path (PKGDATADIR) : %s\n" $(abspath $(PKGDATADIR))
+	@printf "\tshim path (PKGLIBEXECDIR) : %s\n" $(abspath $(PKGLIBEXECDIR))
+	@printf "\n"
 ## help: Show help comments that start with `##VAR` and `##TARGET`
 help: Makefile show-summary
 	@echo "========================== Help ============================="
@@ -429,10 +487,10 @@ codecov-html: check_tarpaulin
 install: install-runtime install-configs
 
 install-runtime: runtime
-	install -D $(TARGET_PATH) $(BINDIR)
+	install -D $(TARGET_PATH) $(DESTDIR)$(BINDIR)/$(notdir $(TARGET_PATH))
 
 install-configs: $(CONFIGS)
-	$(foreach f,$(CONFIGS),$(call INSTALL_CONFIG,$f,$(dir $(CONFIG_PATH)))) \
+	$(foreach f,$(CONFIGS),$(call INSTALL_FILE,$f,$(dir $(CONFIG_PATH)))) \
     sudo ln -sf $(DEFAULT_HYPERVISOR_CONFIG) $(DESTDIR)/$(CONFIG_PATH)
 
 .PHONY: \

src/runtime-rs/README.md

@@ -0,0 +1,126 @@
+# runtime-rs
+
+## Wath's runtime-rs
+
+`runtime-rs` is a new component introduced in Kata Containers 3.0, it is a Rust version of runtime(shim). It like [runtime](../runtime), but they have many difference:
+
+- `runtime-rs` is written in Rust, and `runtime` is written in Go.
+- `runtime` is the default shim in Kata Containers 3.0, `runtime-rs` is still under heavy development.
+- `runtime-rs` has a completed different architecture than `runtime`, you can check at the [architecture overview](../../docs/design/architecture_3.0).
+
+**Note**:
+
+`runtime-rs` is still under heavy development, you should avoid using it in critical system.
+
+## Architecture overview
+
+Also, `runtime-rs` provides the following features:
+
+- Turn key solution with builtin `Dragonball` Sandbox, all components in one process
+- Async I/O to reduce resource consumption
+- Extensible framework for multiple services, runtimes and hypervisors
+- Lifecycle management for sandbox and container associated resources
+
+See the [architecture overview](../../docs/design/architecture_3.0)
+for details on the `runtime-rs` design.
+
+`runtime-rs` is a runtime written in Rust, it is composed of several crates.
+
+This picture shows the overview about the crates under this directory and the relation between crates.
+
+![crates overview](docs/images/crate-overview.svg)
+
+Not all the features have been implemented yet, for details please check the [roadmap](../../docs/design/architecture_3.0/README.md#roadmap).
+
+## Crates
+
+The `runtime-rs` directory contains some crates in the crates directory that compose the `containerd-shim-kata-v2`.
+
+| Crate | Description |
+|-|-|
+| [`shim`](crates/shim)| containerd shimv2 implementation |
+| [`service`](crates/service)| services for containers, includes task service |
+| [`runtimes`](crates/runtimes)| container runtimes |
+| [`resource`](crates/resource)| sandbox and container resources |
+| [`hypervisor`](crates/hypervisor)| hypervisor that act as a sandbox |
+| [`agent`](crates/agent)| library used to communicate with agent in the guest OS |
+| [`persist`](crates/persist)| persist container state to disk |
+
+### shim
+
+`shim` is the entry point of the containerd shim process, it implements containerd shim's [binary protocol](https://github.com/containerd/containerd/tree/v1.6.8/runtime/v2#commands):
+
+- start: start a new shim process
+- delete: delete exist a shim process
+- run: run ttRPC service in shim
+
+containerd will launch a shim process and the shim process will serve as a ttRPC server to provide shim service through `TaskService` from `service` crate.
+
+### service
+
+The `runtime-rs` has an extensible framework, includes extension of services, runtimes, and hypervisors.
+
+Currently, only containerd compatible `TaskService` is implemented.
+
+`TaskService` has implemented the [containerd shim protocol](https://docs.rs/containerd-shim-protos/0.2.0/containerd_shim_protos/),
+and interacts with runtimes through messages.
+
+### runtimes
+
+Runtime is a container runtime, the runtime handler handles messages from task services to manage containers.
+Runtime handler and Runtime instance is used to deal with the operation for sandbox and container.
+
+Currently, only `VirtContainer` has been implemented.
+
+### resource
+
+In `runtime-rs`, all networks/volumes/rootfs are abstracted as resources.
+
+Resources are classified into two types:
+
+- sandbox resources: network, share-fs
+- container resources: rootfs, volume, cgroup
+
+[Here](../../docs/design/architecture_3.0/README.md#resource-manager) is a detailed description of the resources.
+
+### hypervisor
+
+For `VirtContainer`, there will be more hypervisors to choose.
+
+Currently, only built-in `Dragonball` has been implemented.
+
+### agent
+
+`agent` is used to communicate with agent in the guest OS from the shim side. The only supported agent is `KataAgent`.
+
+### persist
+
+Persist defines traits and functions to help different components save state to disk and load state from disk.
+
+## Build and install
+
+```bash
+$ make
+$ sudo make install
+```
+## Configuration
+
+`runtime-rs` has the same [configuration as `runtime`](../runtime/README.md#configuration) with some [limitations](#limitations).
+
+## Logging
+
+See the
+[debugging section of the developer guide](../../docs/Developer-Guide.md#troubleshoot-kata-containers).
+
+## Debugging
+
+See the
+[debugging section of the developer guide](../../docs/Developer-Guide.md#troubleshoot-kata-containers).
+
+## Limitations
+
+For Kata Containers limitations, see the
+[limitations file](../../docs/Limitations.md)
+for further details.
+
+`runtime-rs` is under heavy developments, and doesn't support all features as the Golang version [`runtime`](../runtime), check the [roadmap](../../docs/design/architecture_3.0/README.md#roadmap) for details.

src/runtime-rs/config/configuration-dragonball.toml.in

@@ -128,9 +128,12 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_DB@"
 #guest_hook_path = "/usr/share/oci/hooks"
 
 # Shared file system type:
-#   - virtio-fs (default)
+#   - inline-virtio-fs (default)
+#   - virtio-fs
 #   - virtio-9p
 #   - virtio-fs-nydus
+# "inline-virtio-fs" is the same as "virtio-fs", but it is running in the same process
+# of shim, does not need an external virtiofsd process.
 shared_fs = "@DBSHAREDFS@"
 
 [agent.@PROJECT_TYPE@]
@@ -247,3 +250,6 @@ experimental=@DEFAULTEXPFEATURES@
 # If enabled, user can run pprof tools with shim v2 process through kata-monitor.
 # (default: false)
 # enable_pprof = true
+
+static_resource_mgmt=@DEFSTATICRESOURCEMGMT_DB@
+

src/runtime-rs/crates/agent/Cargo.toml

@@ -3,6 +3,7 @@ name = "agent"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dev-dependencies]
 futures = "0.1.27"

src/runtime-rs/crates/agent/src/kata/agent.rs

@@ -8,6 +8,8 @@ use anyhow::{Context, Result};
 use async_trait::async_trait;
 use ttrpc::context as ttrpc_ctx;
 
+use kata_types::config::Agent as AgentConfig;
+
 use crate::{kata::KataAgent, Agent, AgentManager, HealthService};
 
 /// millisecond to nanosecond
@@ -37,6 +39,10 @@ impl AgentManager for KataAgent {
     async fn stop(&self) {
         self.stop_log_forwarder().await;
     }
+
+    async fn agent_config(&self) -> AgentConfig {
+        self.agent_config().await
+    }
 }
 
 // implement for health service
@@ -90,9 +96,9 @@ impl_agent!(
     stats_container | crate::ContainerID | crate::StatsContainerResponse | None,
     pause_container | crate::ContainerID | crate::Empty | None,
     resume_container | crate::ContainerID | crate::Empty | None,
-    write_stdin | crate::WriteStreamRequest | crate::WriteStreamResponse | None,
-    read_stdout | crate::ReadStreamRequest | crate::ReadStreamResponse | None,
-    read_stderr | crate::ReadStreamRequest | crate::ReadStreamResponse | None,
+    write_stdin | crate::WriteStreamRequest | crate::WriteStreamResponse | Some(0),
+    read_stdout | crate::ReadStreamRequest | crate::ReadStreamResponse | Some(0),
+    read_stderr | crate::ReadStreamRequest | crate::ReadStreamResponse | Some(0),
     close_stdin | crate::CloseStdinRequest | crate::Empty | None,
     tty_win_resize | crate::TtyWinResizeRequest | crate::Empty | None,
     update_interface | crate::UpdateInterfaceRequest | crate::Interface | None,

src/runtime-rs/crates/agent/src/kata/mod.rs

@@ -126,4 +126,9 @@ impl KataAgent {
         let mut inner = self.inner.lock().await;
         inner.log_forwarder.stop();
     }
+
+    pub(crate) async fn agent_config(&self) -> AgentConfig {
+        let inner = self.inner.lock().await;
+        inner.config.clone()
+    }
 }

src/runtime-rs/crates/agent/src/lib.rs

@@ -29,10 +29,16 @@ pub use types::{
 use anyhow::Result;
 use async_trait::async_trait;
 
+use kata_types::config::Agent as AgentConfig;
+
+pub const AGENT_KATA: &str = "kata";
+
 #[async_trait]
 pub trait AgentManager: Send + Sync {
     async fn start(&self, address: &str) -> Result<()>;
     async fn stop(&self);
+
+    async fn agent_config(&self) -> AgentConfig;
 }
 
 #[async_trait]

src/runtime-rs/crates/agent/src/types.rs

@@ -4,6 +4,9 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use anyhow::{anyhow, Result};
+use std::convert::TryFrom;
+
 use serde::Deserialize;
 
 #[derive(PartialEq, Clone, Default)]
@@ -390,6 +393,56 @@ pub struct KernelModule {
     pub parameters: Vec<String>,
 }
 
+impl KernelModule {
+    pub fn set_kernel_modules(modules: Vec<String>) -> Result<Vec<Self>> {
+        let mut kernel_modules = Vec::new();
+        for module_string in modules {
+            if module_string.is_empty() {
+                continue;
+            }
+            let kernel_module = Self::try_from(module_string)?;
+            kernel_modules.push(kernel_module);
+        }
+        Ok(kernel_modules)
+    }
+}
+
+impl TryFrom<String> for KernelModule {
+    type Error = anyhow::Error;
+    // input string: " ModuleName Param1 Param2 ... "
+    // NOTICE: " ModuleName Param1="spaces in here" " => KernelModule { name: ModuleName, parameters: Param1="spaces in here" }
+    fn try_from(str: String) -> Result<Self> {
+        let split: Vec<&str> = str.split(' ').collect();
+        let mut name = String::new();
+        let mut parameters = Vec::new();
+
+        let mut flag = false;
+        for (index, info) in split.iter().enumerate() {
+            if index == 0 {
+                name = info.to_string();
+            } else if flag {
+                // a former param's string contains \"
+                if let Some(former_param) = parameters.pop() {
+                    let cur_param = format!("{} {}", former_param, info);
+                    parameters.push(cur_param);
+                }
+            } else {
+                parameters.push(info.to_string());
+            }
+
+            if info.contains('\"') {
+                flag = !flag;
+            }
+        }
+
+        if flag {
+            return Err(anyhow!("\" not match"));
+        }
+
+        Ok(KernelModule { name, parameters })
+    }
+}
+
 #[derive(PartialEq, Clone, Default)]
 pub struct CreateSandboxRequest {
     pub hostname: String,
@@ -486,3 +539,44 @@ pub struct VersionCheckResponse {
 pub struct OomEventResponse {
     pub container_id: String,
 }
+
+#[cfg(test)]
+mod test {
+    use std::convert::TryFrom;
+
+    use super::KernelModule;
+
+    #[test]
+    fn test_new_kernel_module() {
+        let kernel_module_str1 = "ModuleName Param1 Param2";
+        let kernel_module1 = KernelModule::try_from(kernel_module_str1.to_string()).unwrap();
+        assert!(kernel_module1.name == "ModuleName");
+        assert!(kernel_module1.parameters[0] == "Param1");
+        assert!(kernel_module1.parameters[1] == "Param2");
+
+        let kernel_module_str2 = "ModuleName Param1=\"spaces in here\"";
+        let kernel_module2 = KernelModule::try_from(kernel_module_str2.to_string()).unwrap();
+        assert!(kernel_module2.name == "ModuleName");
+        assert!(kernel_module2.parameters[0] == "Param1=\"spaces in here\"");
+
+        // exception case
+        let kernel_module_str3 = "ModuleName \"Param1";
+        let kernel_module3 = KernelModule::try_from(kernel_module_str3.to_string());
+        assert!(kernel_module3.is_err());
+    }
+
+    #[test]
+    fn test_kernel_modules() {
+        let kernel_module_str1 = "ModuleName1 Param1 Param2".to_string();
+        let kernel_module_str2 = "".to_string();
+        let kernel_module_str3 = "ModuleName2".to_string();
+        let kernel_modules_str = vec![kernel_module_str1, kernel_module_str2, kernel_module_str3];
+
+        let kernel_modules = KernelModule::set_kernel_modules(kernel_modules_str).unwrap();
+        assert!(kernel_modules.len() == 2);
+        assert!(kernel_modules[0].name == "ModuleName1");
+        assert!(kernel_modules[0].parameters.len() == 2);
+        assert!(kernel_modules[1].name == "ModuleName2");
+        assert!(kernel_modules[1].parameters.is_empty());
+    }
+}

src/runtime-rs/crates/hypervisor/Cargo.toml

@@ -3,6 +3,7 @@ name = "hypervisor"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -12,7 +13,7 @@ async-trait = "0.1.48"
 dbs-utils = "0.1.0"
 go-flag = "0.1.0"
 libc = ">=0.2.39"
-nix = "0.24.1"
+nix = "0.24.2"
 persist = { path = "../persist" }
 seccompiler = "0.2.0"
 serde = { version = "1.0.138", features = ["derive"] }

src/runtime-rs/crates/hypervisor/src/dragonball/inner.rs

@@ -176,6 +176,7 @@ impl DragonballInner {
             serial_path: Some(serial_path),
             mem_size_mib: self.config.memory_info.default_memory as usize,
             vcpu_count: self.config.cpu_info.default_vcpus as u8,
+            max_vcpu_count: self.config.cpu_info.default_maxvcpus as u8,
             ..Default::default()
         };
         info!(sl!(), "vm config: {:?}", vm_config);

src/runtime-rs/crates/persist/Cargo.toml

@@ -3,6 +3,7 @@ name = "persist"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 async-trait = "0.1.48"
@@ -10,7 +11,6 @@ anyhow = "^1.0"
 kata-sys-util = { path = "../../../libs/kata-sys-util"}
 kata-types = { path = "../../../libs/kata-types" }
 libc = "0.2"
-rustc-serialize = "0.3.24"
 serde = { version = "1.0.138", features = ["derive"] }
 serde_json = "1.0.82"
 safe-path = { path = "../../../libs/safe-path"}

src/runtime-rs/crates/resource/Cargo.toml

@@ -3,6 +3,7 @@ name = "resource"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 anyhow = "^1.0"
@@ -14,7 +15,7 @@ lazy_static = "1.4.0"
 libc = ">=0.2.39"
 netlink-sys = "0.8.3"
 netlink-packet-route = "0.13.0"
-nix = "0.24.1"
+nix = "0.24.2"
 rand = "^0.7.2"
 rtnetlink = "0.11.0"
 scopeguard = "1.0.0"

src/runtime-rs/crates/resource/src/rootfs/mod.rs

@@ -11,7 +11,6 @@ use std::{sync::Arc, vec::Vec};
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use kata_types::mount::Mount;
-use nix::sys::stat::{self, SFlag};
 use tokio::sync::RwLock;
 
 use crate::share_fs::ShareFs;
@@ -98,24 +97,3 @@ impl RootFsResource {
 fn is_single_layer_rootfs(rootfs_mounts: &[Mount]) -> bool {
     rootfs_mounts.len() == 1
 }
-
-#[allow(dead_code)]
-fn get_block_device(file_path: &str) -> Option<u64> {
-    if file_path.is_empty() {
-        return None;
-    }
-
-    match stat::stat(file_path) {
-        Ok(fstat) => {
-            if SFlag::from_bits_truncate(fstat.st_mode) == SFlag::S_IFBLK {
-                return Some(fstat.st_rdev);
-            }
-        }
-        Err(err) => {
-            error!(sl!(), "failed to stat for {} {:?}", file_path, err);
-            return None;
-        }
-    };
-
-    None
-}

src/runtime-rs/crates/resource/src/volume/share_fs_volume.rs

@@ -121,7 +121,7 @@ fn is_host_device(dest: &str) -> bool {
         return true;
     }
 
-    if dest.starts_with("/dev") {
+    if dest.starts_with("/dev/") {
         let src = match std::fs::canonicalize(dest) {
             Err(_) => return false,
             Ok(src) => src,
@@ -138,7 +138,6 @@ fn is_host_device(dest: &str) -> bool {
 }
 
 // Note, don't generate random name, attaching rafs depends on the predictable name.
-// If template_mnt is passed, just use existed name in it
 pub fn generate_mount_path(id: &str, file_name: &str) -> String {
     let mut nid = String::from(id);
     if nid.len() > 10 {

src/runtime-rs/crates/runtimes/Cargo.toml

@@ -3,6 +3,7 @@ name = "runtimes"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 anyhow = "^1.0"

src/runtime-rs/crates/runtimes/common/Cargo.toml

@@ -3,6 +3,7 @@ name = "common"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -11,7 +12,7 @@ anyhow = "^1.0"
 async-trait = "0.1.48"
 containerd-shim-protos = { version = "0.2.0", features = ["async"]}
 lazy_static = "1.4.0"
-nix = "0.24.1"
+nix = "0.24.2"
 protobuf = "2.27.0"
 serde_json = "1.0.39"
 slog = "2.5.2"

src/runtime-rs/crates/runtimes/common/src/types/mod.rs

@@ -128,6 +128,7 @@ pub struct ContainerConfig {
     pub bundle: String,
     pub rootfs_mounts: Vec<Mount>,
     pub terminal: bool,
+    pub options: Option<Vec<u8>>,
     pub stdin: Option<String>,
     pub stdout: Option<String>,
     pub stderr: Option<String>,

src/runtime-rs/crates/runtimes/common/src/types/trans_from_shim.rs

@@ -4,19 +4,17 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::{
-    convert::{From, TryFrom},
-    path::PathBuf,
-};
-
-use anyhow::{Context, Result};
-use containerd_shim_protos::api;
-use kata_types::mount::Mount;
-
 use super::{
     ContainerConfig, ContainerID, ContainerProcess, ExecProcessRequest, KillRequest, Request,
     ResizePTYRequest, ShutdownRequest, UpdateRequest,
 };
+use anyhow::{Context, Result};
+use containerd_shim_protos::api;
+use kata_types::mount::Mount;
+use std::{
+    convert::{From, TryFrom},
+    path::PathBuf,
+};
 
 fn trans_from_shim_mount(from: api::Mount) -> Mount {
     let options = from.options.to_vec();
@@ -42,6 +40,11 @@ fn trans_from_shim_mount(from: api::Mount) -> Mount {
 impl TryFrom<api::CreateTaskRequest> for Request {
     type Error = anyhow::Error;
     fn try_from(from: api::CreateTaskRequest) -> Result<Self> {
+        let options = if from.has_options() {
+            Some(from.get_options().get_value().to_vec())
+        } else {
+            None
+        };
         Ok(Request::CreateContainer(ContainerConfig {
             container_id: from.id.clone(),
             bundle: from.bundle.clone(),
@@ -52,6 +55,7 @@ impl TryFrom<api::CreateTaskRequest> for Request {
                 .map(trans_from_shim_mount)
                 .collect(),
             terminal: from.terminal,
+            options,
             stdin: (!from.stdin.is_empty()).then(|| from.stdin.clone()),
             stdout: (!from.stdout.is_empty()).then(|| from.stdout.clone()),
             stderr: (!from.stderr.is_empty()).then(|| from.stderr.clone()),

src/runtime-rs/crates/runtimes/src/lib.rs

@@ -11,3 +11,4 @@ logging::logger_with_subsystem!(sl, "runtimes");
 
 pub mod manager;
 pub use manager::RuntimeHandlerManager;
+mod static_resource;

src/runtime-rs/crates/runtimes/src/manager.rs

@@ -4,9 +4,11 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::sync::Arc;
+use std::{str::from_utf8, sync::Arc};
 
 use anyhow::{anyhow, Context, Result};
+
+use crate::static_resource::StaticResourceManager;
 use common::{
     message::Message,
     types::{Request, Response},
@@ -72,7 +74,7 @@ impl RuntimeHandlerManagerInner {
         Ok(())
     }
 
-    async fn try_init(&mut self, spec: &oci::Spec) -> Result<()> {
+    async fn try_init(&mut self, spec: &oci::Spec, options: &Option<Vec<u8>>) -> Result<()> {
         // return if runtime instance has init
         if self.runtime_instance.is_some() {
             return Ok(());
@@ -102,7 +104,7 @@ impl RuntimeHandlerManagerInner {
             None
         };
 
-        let config = load_config(spec).context("load config")?;
+        let config = load_config(spec, options).context("load config")?;
         self.init_runtime_handler(netns, Arc::new(config))
             .await
             .context("init runtime handler")?;
@@ -170,9 +172,13 @@ impl RuntimeHandlerManager {
             .ok_or_else(|| anyhow!("runtime not ready"))
     }
 
-    async fn try_init_runtime_instance(&self, spec: &oci::Spec) -> Result<()> {
+    async fn try_init_runtime_instance(
+        &self,
+        spec: &oci::Spec,
+        options: &Option<Vec<u8>>,
+    ) -> Result<()> {
         let mut inner = self.inner.write().await;
-        inner.try_init(spec).await
+        inner.try_init(spec, options).await
     }
 
     pub async fn handler_message(&self, req: Request) -> Result<Response> {
@@ -181,7 +187,7 @@ impl RuntimeHandlerManager {
             let bundler_path = format!("{}/{}", req.bundle, oci::OCI_SPEC_CONFIG_FILE_NAME);
             let spec = oci::Spec::load(&bundler_path).context("load spec")?;
 
-            self.try_init_runtime_instance(&spec)
+            self.try_init_runtime_instance(&spec, &req.options)
                 .await
                 .context("try init runtime instance")?;
             let instance = self
@@ -296,13 +302,21 @@ impl RuntimeHandlerManager {
 /// 2. shimv2 create task option
 /// TODO: https://github.com/kata-containers/kata-containers/issues/3961
 /// 3. environment
-fn load_config(spec: &oci::Spec) -> Result<TomlConfig> {
+fn load_config(spec: &oci::Spec, option: &Option<Vec<u8>>) -> Result<TomlConfig> {
     const KATA_CONF_FILE: &str = "KATA_CONF_FILE";
     let annotation = Annotation::new(spec.annotations.clone());
     let config_path = if let Some(path) = annotation.get_sandbox_config_path() {
         path
     } else if let Ok(path) = std::env::var(KATA_CONF_FILE) {
         path
+    } else if let Some(option) = option {
+        // get rid of the special characters in options to get the config path
+        let path = if option.len() > 2 {
+            from_utf8(&option[2..])?.to_string()
+        } else {
+            String::from("")
+        };
+        path
     } else {
         String::from("")
     };
@@ -310,6 +324,23 @@ fn load_config(spec: &oci::Spec) -> Result<TomlConfig> {
     let (mut toml_config, _) =
         TomlConfig::load_from_file(&config_path).context("load toml config")?;
     annotation.update_config_by_annotation(&mut toml_config)?;
+
+    // Sandbox sizing information *may* be provided in two scenarios:
+    //   1. The upper layer runtime (ie, containerd or crio) provide sandbox sizing information as an annotation
+    //	in the 'sandbox container's' spec. This would typically be a scenario where as part of a create sandbox
+    //	request the upper layer runtime receives this information as part of a pod, and makes it available to us
+    //	for sizing purposes.
+    //   2. If this is not a sandbox infrastructure container, but instead a standalone single container (analogous to "docker run..."),
+    //	then the container spec itself will contain appropriate sizing information for the entire sandbox (since it is
+    //	a single container.
+    if toml_config.runtime.static_resource_mgmt {
+        info!(sl!(), "static resource management enabled");
+        let static_resource_manager = StaticResourceManager::new(spec)
+            .context("failed to construct static resource manager")?;
+        static_resource_manager
+            .setup_config(&mut toml_config)
+            .context("failed to setup static resource mgmt config")?;
+    }
     info!(sl!(), "get config content {:?}", &toml_config);
     Ok(toml_config)
 }

src/runtime-rs/crates/runtimes/src/static_resource.rs

@@ -0,0 +1,167 @@
+// Copyright (c) 2019-2021 Alibaba Cloud
+// Copyright (c) 2019-2021 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+use std::convert::TryFrom;
+
+use anyhow::{Context, Result};
+
+use kata_types::{
+    annotations::Annotation, config::TomlConfig, container::ContainerType,
+    cpu::LinuxContainerCpuResources, k8s::container_type,
+};
+
+// static resource that StaticResourceManager needs, this is the spec for the
+// sandbox/container's workload
+#[derive(Clone, Copy, Debug)]
+struct StaticResource {
+    vcpu: u32,
+    mem_mb: u32,
+}
+
+// generate static resource(vcpu and memory in MiB) from spec's information
+// used for static resource management
+impl TryFrom<&oci::Spec> for StaticResource {
+    type Error = anyhow::Error;
+    fn try_from(spec: &oci::Spec) -> Result<Self> {
+        let mut vcpu: u32 = 0;
+        let mut mem_mb: u32 = 0;
+        match container_type(spec) {
+            // podsandbox, from annotation
+            ContainerType::PodSandbox => {
+                let annotation = Annotation::new(spec.annotations.clone());
+                let (period, quota, memory) =
+                    get_sizing_info(annotation).context("failed to get sizing info")?;
+                let cpu = oci::LinuxCpu {
+                    period: Some(period),
+                    quota: Some(quota),
+                    ..Default::default()
+                };
+                // although it may not be actually a linux container, we are only using the calculation inside
+                // LinuxContainerCpuResources::try_from to generate our vcpu number
+                if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(&cpu) {
+                    vcpu = get_nr_vcpu(&cpu_resource);
+                }
+                mem_mb = convert_memory_to_mb(memory);
+            }
+            // single container, from container spec
+            _ => {
+                if let Some(linux) = &spec.linux {
+                    if let Some(resource) = &linux.resources {
+                        if let Some(cpu) = &resource.cpu {
+                            if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(cpu) {
+                                vcpu = get_nr_vcpu(&cpu_resource);
+                            }
+                        }
+                        if let Some(mem) = &resource.memory {
+                            let memory = mem.limit.unwrap_or(0);
+                            mem_mb = convert_memory_to_mb(memory);
+                        }
+                    }
+                }
+            }
+        }
+        info!(
+            sl!(),
+            "static resource mgmt result: vcpu={}, mem_mb={}", vcpu, mem_mb
+        );
+        Ok(Self { vcpu, mem_mb })
+    }
+}
+
+// StaticResourceManager is responsible for static resource management
+//
+// static resource management sizing information is optionally provided, either by
+// upper layer runtime (containerd / crio) or by the container spec itself (when it
+// is a standalone single container such as the one started with *docker run*)
+//
+// the sizing information uses three values, cpu quota, cpu period and memory limit,
+// and with above values it calculates the # vcpus and memory for the workload and
+// add them to default value of the config
+#[derive(Clone, Copy, Debug)]
+pub struct StaticResourceManager {
+    resource: StaticResource,
+}
+
+impl StaticResourceManager {
+    pub fn new(spec: &oci::Spec) -> Result<Self> {
+        Ok(Self {
+            resource: StaticResource::try_from(spec)
+                .context("failed to construct static resource")?,
+        })
+    }
+
+    pub fn setup_config(&self, config: &mut TomlConfig) -> Result<()> {
+        // update this data to the hypervisor config for later use by hypervisor
+        let hypervisor_name = &config.runtime.hypervisor_name;
+        let mut hv = config
+            .hypervisor
+            .get_mut(hypervisor_name)
+            .context("failed to get hypervisor config")?;
+        hv.cpu_info.default_vcpus += self.resource.vcpu as i32;
+        hv.memory_info.default_memory += self.resource.mem_mb;
+        Ok(())
+    }
+}
+
+fn get_nr_vcpu(resource: &LinuxContainerCpuResources) -> u32 {
+    if let Some(v) = resource.get_vcpus() {
+        v as u32
+    } else {
+        0
+    }
+}
+
+fn convert_memory_to_mb(memory_in_byte: i64) -> u32 {
+    if memory_in_byte < 0 {
+        0
+    } else {
+        (memory_in_byte / 1024 / 1024) as u32
+    }
+}
+
+// from the upper layer runtime's annotation (e.g. crio, k8s), get the *cpu quota,
+// cpu period and memory limit* for a sandbox/container
+fn get_sizing_info(annotation: Annotation) -> Result<(u64, i64, i64)> {
+    // since we are *adding* our result to the config, a value of 0 will cause no change
+    // and if the annotation is not assigned (but static resource management is), we will
+    // log a *warning* to fill that with zero value
+    let period = annotation.get_sandbox_cpu_period();
+    let quota = annotation.get_sandbox_cpu_quota();
+    let memory = annotation.get_sandbox_mem();
+    Ok((period, quota, memory))
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_static_resource_mgmt() {
+        // data below should result in 2200 mCPU(round up to 3 vcpus) and 512 MiB of memory
+        let period: u64 = 100000;
+        let memory: i64 = 1048576 * 512; // 512 MiB
+        let quota: i64 = 220000;
+
+        let cpu = oci::LinuxCpu {
+            period: Some(period),
+            quota: Some(quota),
+            ..Default::default()
+        };
+        if let Ok(cpu_resource) = LinuxContainerCpuResources::try_from(&cpu) {
+            if let Some(v) = cpu_resource.get_vcpus() {
+                assert_eq!(v, 3);
+            }
+        }
+
+        let mem_mb = if memory < 0 {
+            0
+        } else {
+            (memory / 1024 / 1024) as u32
+        };
+
+        assert_eq!(mem_mb, 512);
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/Cargo.toml

@@ -3,6 +3,7 @@ name = "virt_container"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 anyhow = "^1.0"
@@ -12,7 +13,7 @@ containerd-shim-protos = { version = "0.2.0", features = ["async"]}
 futures = "0.3.19"
 lazy_static = "1.4.0"
 libc = ">=0.2.39"
-nix = "0.16.0"
+nix = "0.24.2"
 protobuf = "2.27.0"
 serde = { version = "1.0.100", features = ["derive"] }
 serde_derive = "1.0.27"
@@ -22,7 +23,7 @@ slog-scope = "4.4.0"
 tokio = { version = "1.8.0" }
 toml = "0.4.2"
 url = "2.1.1"
-async-std = "0.99.5"
+async-std = "1.12.0"
 
 agent = { path = "../../agent" }
 common = { path = "../common" }

src/runtime-rs/crates/runtimes/virt_container/src/container_manager/container.rs

@@ -80,8 +80,9 @@ impl Container {
         let mut inner = self.inner.write().await;
         let toml_config = self.resource_manager.config().await;
         let config = &self.config;
-        let sandbox_pidns = amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp)
-            .context("load spec")?;
+        amend_spec(&mut spec, toml_config.runtime.disable_guest_seccomp).context("amend spec")?;
+        let sandbox_pidns = is_pid_namespace_enabled(&spec);
+
         // handler rootfs
         let rootfs = self
             .resource_manager
@@ -373,7 +374,7 @@ impl Container {
     }
 }
 
-fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<bool> {
+fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<()> {
     // hook should be done on host
     spec.hooks = None;
 
@@ -401,33 +402,29 @@ fn amend_spec(spec: &mut oci::Spec, disable_guest_seccomp: bool) -> Result<bool>
         }
 
         linux.namespaces = ns;
-
-        return Ok(handle_pid_namespace(&linux.namespaces));
     }
 
-    Ok(false)
+    Ok(())
 }
 
-// handle_pid_namespace checks if Pid namespace for a container needs to be shared with its sandbox
+// is_pid_namespace_enabled checks if Pid namespace for a container needs to be shared with its sandbox
 // pid namespace.
-fn handle_pid_namespace(namespaces: &[oci::LinuxNamespace]) -> bool {
-    for n in namespaces.iter() {
-        match n.r#type.as_str() {
-            oci::PIDNAMESPACE => {
-                if !n.path.is_empty() {
-                    return true;
-                }
+fn is_pid_namespace_enabled(spec: &oci::Spec) -> bool {
+    if let Some(linux) = spec.linux.as_ref() {
+        for n in linux.namespaces.iter() {
+            if n.r#type.as_str() == oci::PIDNAMESPACE {
+                return !n.path.is_empty();
             }
-            _ => continue,
         }
     }
+
     false
 }
 
 #[cfg(test)]
 mod tests {
     use super::amend_spec;
-    use crate::container_manager::container::handle_pid_namespace;
+    use super::is_pid_namespace_enabled;
     #[test]
     fn test_amend_spec_disable_guest_seccomp() {
         let mut spec = oci::Spec {
@@ -448,38 +445,69 @@ mod tests {
         amend_spec(&mut spec, true).unwrap();
         assert!(spec.linux.as_ref().unwrap().seccomp.is_none());
     }
+
     #[test]
-    fn test_handle_pid_namespace() {
-        let namespaces = vec![
-            oci::LinuxNamespace {
-                r#type: "pid".to_string(),
-                path: "".to_string(),
+    fn test_is_pid_namespace_enabled() {
+        struct TestData<'a> {
+            desc: &'a str,
+            namespaces: Vec<oci::LinuxNamespace>,
+            result: bool,
+        }
+
+        let tests = &[
+            TestData {
+                desc: "no pid namespace",
+                namespaces: vec![oci::LinuxNamespace {
+                    r#type: "network".to_string(),
+                    path: "".to_string(),
+                }],
+                result: false,
             },
-            oci::LinuxNamespace {
-                r#type: "network".to_string(),
-                path: "".to_string(),
+            TestData {
+                desc: "empty pid namespace path",
+                namespaces: vec![
+                    oci::LinuxNamespace {
+                        r#type: "pid".to_string(),
+                        path: "".to_string(),
+                    },
+                    oci::LinuxNamespace {
+                        r#type: "network".to_string(),
+                        path: "".to_string(),
+                    },
+                ],
+                result: false,
             },
-            oci::LinuxNamespace {
-                r#type: "ipc".to_string(),
-                path: "".to_string(),
-            },
-            oci::LinuxNamespace {
-                r#type: "uts".to_string(),
-                path: "".to_string(),
-            },
-            oci::LinuxNamespace {
-                r#type: "mount".to_string(),
-                path: "".to_string(),
-            },
-            oci::LinuxNamespace {
-                r#type: "user".to_string(),
-                path: "".to_string(),
-            },
-            oci::LinuxNamespace {
-                r#type: "cgroup".to_string(),
-                path: "".to_string(),
+            TestData {
+                desc: "pid namespace is set",
+                namespaces: vec![
+                    oci::LinuxNamespace {
+                        r#type: "pid".to_string(),
+                        path: "/some/path".to_string(),
+                    },
+                    oci::LinuxNamespace {
+                        r#type: "network".to_string(),
+                        path: "".to_string(),
+                    },
+                ],
+                result: true,
             },
         ];
-        assert!(!handle_pid_namespace(&namespaces));
+
+        let mut spec = oci::Spec::default();
+
+        for (i, d) in tests.iter().enumerate() {
+            spec.linux = Some(oci::Linux {
+                namespaces: d.namespaces.clone(),
+                ..Default::default()
+            });
+
+            assert_eq!(
+                d.result,
+                is_pid_namespace_enabled(&spec),
+                "test[{}]: {:?}",
+                i,
+                d.desc
+            );
+        }
     }
 }

src/runtime-rs/crates/runtimes/virt_container/src/health_check.rs

@@ -91,7 +91,7 @@ impl HealthCheck {
                                         ::std::process::exit(1);
                                     }
                                 } else {
-                                    info!(sl!(), "wait to exit exit {}", id);
+                                    info!(sl!(), "wait to exit {}", id);
                                     break;
                                 }
                             }

src/runtime-rs/crates/runtimes/virt_container/src/lib.rs

@@ -16,7 +16,7 @@ pub mod sandbox_persist;
 
 use std::sync::Arc;
 
-use agent::kata::KataAgent;
+use agent::{kata::KataAgent, AGENT_KATA};
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use common::{message::Message, RuntimeHandler, RuntimeInstance};
@@ -55,20 +55,7 @@ impl RuntimeHandler for VirtContainer {
         let hypervisor = new_hypervisor(&config).await.context("new hypervisor")?;
 
         // get uds from hypervisor and get config from toml_config
-        let agent = Arc::new(KataAgent::new(kata_types::config::Agent {
-            debug: true,
-            enable_tracing: false,
-            server_port: 1024,
-            log_port: 1025,
-            dial_timeout_ms: 10,
-            reconnect_timeout_ms: 3_000,
-            request_timeout_ms: 30_000,
-            health_check_request_timeout_ms: 90_000,
-            kernel_modules: Default::default(),
-            container_pipe_size: 0,
-            debug_console_enabled: false,
-        }));
-
+        let agent = new_agent(&config).context("new agent")?;
         let resource_manager = Arc::new(ResourceManager::new(
             sid,
             agent.clone(),
@@ -121,3 +108,44 @@ async fn new_hypervisor(toml_config: &TomlConfig) -> Result<Arc<dyn Hypervisor>>
         _ => Err(anyhow!("Unsupported hypervisor {}", &hypervisor_name)),
     }
 }
+
+fn new_agent(toml_config: &TomlConfig) -> Result<Arc<KataAgent>> {
+    let agent_name = &toml_config.runtime.agent_name;
+    let agent_config = toml_config
+        .agent
+        .get(agent_name)
+        .ok_or_else(|| anyhow!("failed to get agent for {}", &agent_name))
+        .context("get agent")?;
+    match agent_name.as_str() {
+        AGENT_KATA => {
+            let agent = KataAgent::new(agent_config.clone());
+            Ok(Arc::new(agent))
+        }
+        _ => Err(anyhow!("Unsupported agent {}", &agent_name)),
+    }
+}
+
+#[cfg(test)]
+mod test {
+
+    use super::*;
+
+    fn default_toml_config_agent() -> Result<TomlConfig> {
+        let config_content = r#"
+[agent.kata]
+container_pipe_size=1
+
+[runtime]
+agent_name="kata"
+        "#;
+        TomlConfig::load(config_content).map_err(|e| anyhow!("can not load config toml: {}", e))
+    }
+
+    #[test]
+    fn test_new_agent() {
+        let toml_config = default_toml_config_agent().unwrap();
+
+        let res = new_agent(&toml_config);
+        assert!(res.is_ok());
+    }
+}

src/runtime-rs/crates/runtimes/virt_container/src/sandbox.rs

@@ -6,7 +6,7 @@
 
 use std::sync::Arc;
 
-use agent::{self, kata::KataAgent, Agent};
+use agent::{self, kata::KataAgent, types::KernelModule, Agent};
 use anyhow::{anyhow, Context, Result};
 use async_trait::async_trait;
 use common::{
@@ -159,6 +159,8 @@ impl Sandbox for VirtSandbox {
             .context("setup device after start vm")?;
 
         // create sandbox in vm
+        let agent_config = self.agent.agent_config().await;
+        let kernel_modules = KernelModule::set_kernel_modules(agent_config.kernel_modules)?;
         let req = agent::CreateSandboxRequest {
             hostname: "".to_string(),
             dns: vec![],
@@ -175,7 +177,7 @@ impl Sandbox for VirtSandbox {
                 .await
                 .security_info
                 .guest_hook_path,
-            kernel_modules: vec![],
+            kernel_modules,
         };
 
         self.agent

src/runtime-rs/crates/service/Cargo.toml

@@ -3,6 +3,7 @@ name = "service"
 version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
+license = "Apache-2.0"
 
 [dependencies]
 anyhow = "^1.0"

src/runtime-rs/crates/shim/Cargo.toml

@@ -19,7 +19,7 @@ containerd-shim-protos = { version = "0.2.0", features = ["async"]}
 go-flag = "0.1.0"
 libc = "0.2.108"
 log = "0.4.14"
-nix = "0.24.1"
+nix = "0.24.2"
 protobuf = "2.27.0"
 sha2 = "=0.9.3"
 slog = {version = "2.5.2", features = ["std", "release_max_level_trace", "max_level_trace"]}

src/runtime-rs/crates/shim/src/args.rs

@@ -15,7 +15,7 @@ use crate::Error;
 /// from a shimv2 container manager such as containerd.
 ///
 /// For detailed information, please refer to the
-/// [shim spec](https://github.com/containerd/containerd/blob/main/runtime/v2/README.md).
+/// [shim spec](https://github.com/containerd/containerd/blob/v1.6.8/runtime/v2/README.md).
 #[derive(Debug, Default, Clone)]
 pub struct Args {
     /// the id of the container
@@ -26,8 +26,6 @@ pub struct Args {
     pub address: String,
     /// the binary path to publish events back to containerd
     pub publish_binary: String,
-    ///  Abstract socket path to serve.
-    pub socket: String,
     /// the path to the bundle to delete
     pub bundle: String,
     /// Whether or not to enable debug
@@ -123,7 +121,6 @@ mod tests {
         let default_namespace = "ns1".to_string();
         let default_address = bind_address.to_string();
         let default_publish_binary = "containerd".to_string();
-        let default_socket = "socket".to_string();
         let default_bundle = path.to_string();
         let default_debug = false;
 
@@ -132,7 +129,6 @@ mod tests {
             namespace: default_namespace.clone(),
             address: default_address.clone(),
             publish_binary: default_publish_binary.clone(),
-            socket: default_socket,
             bundle: default_bundle.clone(),
             debug: default_debug,
         };

src/runtime-rs/crates/shim/src/bin/main.rs

@@ -43,7 +43,6 @@ fn parse_args(args: &[OsString]) -> Result<Action> {
         flags.add_flag("id", &mut shim_args.id);
         flags.add_flag("namespace", &mut shim_args.namespace);
         flags.add_flag("publish-binary", &mut shim_args.publish_binary);
-        flags.add_flag("socket", &mut shim_args.socket);
         flags.add_flag("help", &mut help);
         flags.add_flag("version", &mut version);
     })
@@ -87,8 +86,6 @@ fn show_help(cmd: &OsStr) {
         namespace that owns the shim
   -publish-binary string
         path to publish binary (used for publishing events) (default "containerd")
-  -socket string
-        socket path to serve
   --version
         show the runtime version detail and exit
 "#,

src/runtime-rs/crates/shim/src/config.rs.in

@@ -7,8 +7,6 @@
 // WARNING: This file is auto-generated - DO NOT EDIT!
 //
 
-#![allow(dead_code)]
-
 pub const PROJECT_NAME: &str = "@PROJECT_NAME@";
 pub const RUNTIME_VERSION: &str = "@RUNTIME_VERSION@";
 pub const RUNTIME_VERSION_COMMIT: &str = "@VERSION_COMMIT@";

src/runtime-rs/crates/shim/src/core_sched.rs

@@ -0,0 +1,102 @@
+// Copyright (c) 2019-2022 Alibaba Cloud
+// Copyright (c) 2019-2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+//
+// Core Scheduling landed in linux 5.14, this enables that -
+// ONLY the processes have the same cookie value can share an SMT core for security
+// reasons, since SMT siblings share their cpu caches and many other things. This can
+// prevent some malicious processes steal others' private information.
+//
+// This is enabled by containerd, see https://github.com/containerd/containerd/blob/main/docs/man/containerd-config.toml.5.md#format
+//
+// This is done by using system call prctl(), for core scheduling purpose, it is defined as
+// int prctl(PR_SCHED_CORE, int cs_command, pid_t pid, enum pid_type type,
+//           unsigned long *cookie);
+//
+// You may go to https://lwn.net/Articles/861251/, https://lore.kernel.org/lkml/20210422123309.039845339@infradead.org/
+// and kernel.org/doc/html/latest/admin-guide/hw-vuln/core-scheduling.html for more info.
+//
+
+use anyhow::Result;
+use nix::{self, errno::Errno};
+
+#[allow(dead_code)]
+pub const PID_GROUP: usize = 0;
+#[allow(dead_code)]
+pub const THREAD_GROUP: usize = 1;
+pub const PROCESS_GROUP: usize = 2;
+
+#[allow(dead_code)]
+pub const PR_SCHED_CORE: i32 = 62;
+pub const PR_SCHED_CORE_CREATE: usize = 1;
+pub const PR_SCHED_CORE_SHARE_FROM: usize = 3;
+
+// create a new core sched domain, this will NOT succeed if kernel version < 5.14
+pub fn core_sched_create(pidtype: usize) -> Result<(), Errno> {
+    let errno = unsafe { nix::libc::prctl(PR_SCHED_CORE, PR_SCHED_CORE_CREATE, 0, pidtype, 0) };
+    if errno != 0 {
+        Err(nix::errno::Errno::from_i32(-errno))
+    } else {
+        Ok(())
+    }
+}
+
+// shares the domain with *pid*
+#[allow(dead_code)]
+pub fn core_sched_share_from(pid: usize, pidtype: usize) -> Result<(), Errno> {
+    let errno =
+        unsafe { nix::libc::prctl(PR_SCHED_CORE, PR_SCHED_CORE_SHARE_FROM, pid, pidtype, 0) };
+    if errno != 0 {
+        Err(nix::errno::Errno::from_i32(-errno))
+    } else {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use nix::errno::Errno::{EINVAL, ENODEV, ENOMEM, EPERM, ESRCH};
+
+    const RELEASE_MAJOR_VERSION: u8 = 5;
+    const RELEASE_MINOR_VERSION: u8 = 14;
+
+    // since this feature only lands in linux 5.14, we run the test when version is higher
+    fn core_sched_landed() -> bool {
+        let vinfo = std::fs::read_to_string("/proc/sys/kernel/osrelease");
+        if let Ok(info) = vinfo {
+            let vnum: Vec<&str> = info.as_str().split('.').collect();
+            if vnum.len() >= 2 {
+                let major: u8 = vnum[0].parse().unwrap();
+                let minor: u8 = vnum[1].parse().unwrap();
+                return major >= RELEASE_MAJOR_VERSION && minor >= RELEASE_MINOR_VERSION;
+            }
+        }
+        false
+    }
+
+    #[test]
+    fn test_core_sched() {
+        std::env::set_var("SCHED_CORE", "1");
+        assert_eq!(std::env::var("SCHED_CORE").unwrap(), "1");
+        if core_sched_landed() {
+            // it is possible that the machine running this test does not support SMT,
+            // therefore it does not make sense to assert a successful prctl call
+            // but we can still make sure that the return value is a possible value
+            let e = core_sched_create(PROCESS_GROUP);
+            if let Err(errno) = e {
+                if errno != EINVAL
+                    && errno != ENODEV
+                    && errno != ENOMEM
+                    && errno != EPERM
+                    && errno != ESRCH
+                {
+                    panic!("impossible return value {:?}", errno);
+                }
+            }
+        }
+    }
+}

src/runtime-rs/crates/shim/src/lib.rs

@@ -17,6 +17,7 @@ mod logger;
 mod panic_hook;
 mod shim;
 pub use shim::ShimExecutor;
+mod core_sched;
 #[rustfmt::skip]
 pub mod config;
 mod shim_delete;