Merge pull request #3402 from snir911/2.3.1-branch-bump

# Kata Containers 2.3.1

Makefile

@@ -17,16 +17,11 @@ TOOLS += agent-ctl
 
 STANDARD_TARGETS = build check clean install test vendor
 
-default: all
-
-all: logging-crate-tests build
-
-logging-crate-tests:
-	make -C pkg/logging
-
 include utils.mk
 include ./tools/packaging/kata-deploy/local-build/Makefile
 
+all: build
+
 # Create the rules
 $(eval $(call create_all_rules,$(COMPONENTS),$(TOOLS),$(STANDARD_TARGETS)))
 
@@ -39,10 +34,4 @@ generate-protocols:
 static-checks: build
 	bash ci/static-checks.sh
 
-.PHONY: \
-	all \
-	binary-tarball \
-	default \
-	install-binary-tarball \
-	logging-crate-tests \
-	static-checks
+.PHONY: all default static-checks binary-tarball install-binary-tarball

VERSION

@@ -1 +1 @@
-2.4.0-alpha0
+2.3.1

ci/openshift-ci/images/Dockerfile.buildroot

@@ -6,4 +6,9 @@
 #
 FROM registry.centos.org/centos:8
 
-RUN yum -y update && yum -y install git sudo wget
+RUN yum -y update && \
+    yum -y install \
+    git \
+    sudo \
+    wget && \
+    yum clean all

docs/README.md

@@ -52,18 +52,6 @@ Documents that help to understand and contribute to Kata Containers.
 * [How to contribute to Kata Containers](https://github.com/kata-containers/community/blob/master/CONTRIBUTING.md)
 * [Code of Conduct](../CODE_OF_CONDUCT.md)
 
-## Help Writing a Code PR
-
-* [Code PR advice](code-pr-advice.md).
-
-## Help Writing Unit Tests
-
-* [Unit Test Advice](Unit-Test-Advice.md)
-
-## Help Improving the Documents
-
-* [Documentation Requirements](Documentation-Requirements.md)
-
 ### Code Licensing
 
 * [Licensing](Licensing-strategy.md): About the licensing strategy of Kata Containers.
@@ -73,6 +61,10 @@ Documents that help to understand and contribute to Kata Containers.
 * [Release strategy](Stable-Branch-Strategy.md)
 * [Release Process](Release-Process.md)
 
+## Help Improving the Documents
+
+* [Documentation Requirements](Documentation-Requirements.md)
+
 ## Website Changes
 
 If you have a suggestion for how we can improve the

docs/Unit-Test-Advice.md

@@ -1,379 +0,0 @@
-# Unit Test Advice
-
-## Overview
-
-This document offers advice on writing a Unit Test (UT) in
-[Golang](https://golang.org) and [Rust](https://www.rust-lang.org).
-
-## General advice
-
-### Unit test strategies
-
-#### Positive and negative tests
-
-Always add positive tests (where success is expected) *and* negative
-tests (where failure is expected).
-
-#### Boundary condition tests
-
-Try to add unit tests that exercise boundary conditions such as:
-
-- Missing values (`null` or `None`).
-- Empty strings and huge strings.
-- Empty (or uninitialised) complex data structures
-  (such as lists, vectors and hash tables).
-- Common numeric values (such as `-1`, `0`, `1` and the minimum and
-  maximum values).
-
-#### Test unusual values
-
-Also always consider "unusual" input values such as:
-
-- String values containing spaces, Unicode characters, special
-  characters, escaped characters or null bytes.
-
-  > **Note:** Consider these unusual values in prefix, infix and
-  > suffix position.
-
-- String values that cannot be converted into numeric values or which
-  contain invalid structured data (such as invalid JSON).
-
-#### Other types of tests
-
-If the code requires other forms of testing (such as stress testing,
-fuzz testing and integration testing), raise a GitHub issue and
-reference it on the issue you are using for the main work. This
-ensures the test team are aware that a new test is required.
-
-### Test environment
-
-#### Create unique files and directories
-
-Ensure your tests do not write to a fixed file or directory. This can
-cause problems when running multiple tests simultaneously and also
-when running tests after a previous test run failure.
-
-#### Assume parallel testing
-
-Always assume your tests will be run *in parallel*. If this is
-problematic for a test, force it to run in isolation using the
-`serial_test` crate for Rust code for example.
-
-### Running
-
-Ensure you run the unit tests and they all pass before raising a PR.
-Ideally do this on different distributions on different architectures
-to maximise coverage (and so minimise surprises when your code runs in
-the CI).
-
-## Assertions
-
-### Golang assertions
-
-Use the `testify` assertions package to create a new assertion object as this
-keeps the test code free from distracting `if` tests:
-
-```go
-func TestSomething(t *testing.T) {
-    assert := assert.New(t)
-
-    err := doSomething()
-    assert.NoError(err)
-}
-```
-
-### Rust assertions
-
-Use the standard set of `assert!()` macros.
-
-## Table driven tests
-
-Try to write tests using a table-based approach. This allows you to distill
-the logic into a compact table (rather than spreading the tests across
-multiple test functions). It also makes it easy to cover all the
-interesting boundary conditions:
-
-### Golang table driven tests
-
-Assume the following function:
-
-```go
-// The function under test.
-//
-// Accepts a string and an integer and returns the
-// result of sticking them together separated by a dash as a string.
-func joinParamsWithDash(str string, num int) (string, error) {
-    if str == "" {
-        return "", errors.New("string cannot be blank")
-    }
-
-    if num <= 0 {
-        return "", errors.New("number must be positive")
-    }
-
-    return fmt.Sprintf("%s-%d", str, num), nil
-}
-```
-
-A table driven approach to testing it:
-
-```go
-import (
-    "testing"
-    "github.com/stretchr/testify/assert"
-)
-
-func TestJoinParamsWithDash(t *testing.T) {
-    assert := assert.New(t)
-
-    // Type used to hold function parameters and expected results.
-    type testData struct {
-        param1         string
-        param2         int
-        expectedResult string
-        expectError    bool
-    }
-
-    // List of tests to run including the expected results
-    data := []testData{
-        // Failure scenarios
-        {"", -1, "", true},
-        {"", 0, "", true},
-        {"", 1, "", true},
-        {"foo", 0, "", true},
-        {"foo", -1, "", true},
-
-        // Success scenarios
-        {"foo", 1, "foo-1", false},
-        {"bar", 42, "bar-42", false},
-    }
-
-    // Run the tests
-    for i, d := range data {
-        // Create a test-specific string that is added to each assert
-        // call. It will be displayed if any assert test fails.
-        msg := fmt.Sprintf("test[%d]: %+v", i, d)
-
-        // Call the function under test
-        result, err := joinParamsWithDash(d.param1, d.param2)
-
-        // update the message for more information on failure
-        msg = fmt.Sprintf("%s, result: %q, err: %v", msg, result, err)
-
-        if d.expectError {
-            assert.Error(err, msg)
-
-            // If an error is expected, there is no point
-            // performing additional checks.
-            continue
-        }
-
-        assert.NoError(err, msg)
-        assert.Equal(d.expectedResult, result, msg)
-    }
-}
-```
-
-### Rust table driven tests
-
-Assume the following function:
-
-```rust
-// Convenience type to allow Result return types to only specify the type
-// for the true case; failures are specified as static strings.
-// XXX: This is an example. In real code use the "anyhow" and
-// XXX: "thiserror" crates.
-pub type Result<T> = std::result::Result<T, &'static str>;
-
-// The function under test.
-//
-// Accepts a string and an integer and returns the
-// result of sticking them together separated by a dash as a string.
-fn join_params_with_dash(str: &str, num: i32) -> Result<String> {
-    if str.is_empty() {
-        return Err("string cannot be blank");
-    }
-
-    if num <= 0 {
-        return Err("number must be positive");
-    }
-
-    let result = format!("{}-{}", str, num);
-
-    Ok(result)
-}
-
-```
-
-A table driven approach to testing it:
-
-```rust
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_join_params_with_dash() {
-        // This is a type used to record all details of the inputs
-        // and outputs of the function under test.
-        #[derive(Debug)]
-        struct TestData<'a> {
-            str: &'a str,
-            num: i32,
-            result: Result<String>,
-        }
-
-        // The tests can now be specified as a set of inputs and outputs
-        let tests = &[
-            // Failure scenarios
-            TestData {
-                str: "",
-                num: 0,
-                result: Err("string cannot be blank"),
-            },
-            TestData {
-                str: "foo",
-                num: -1,
-                result: Err("number must be positive"),
-            },
-
-            // Success scenarios
-            TestData {
-                str: "foo",
-                num: 42,
-                result: Ok("foo-42".to_string()),
-            },
-            TestData {
-                str: "-",
-                num: 1,
-                result: Ok("--1".to_string()),
-            },
-        ];
-
-        // Run the tests
-        for (i, d) in tests.iter().enumerate() {
-            // Create a string containing details of the test
-            let msg = format!("test[{}]: {:?}", i, d);
-
-            // Call the function under test
-            let result = join_params_with_dash(d.str, d.num);
-
-            // Update the test details string with the results of the call
-            let msg = format!("{}, result: {:?}", msg, result);
-
-            // Perform the checks
-            if d.result.is_ok() {
-                assert!(result == d.result, msg);
-                continue;
-            }
-
-            let expected_error = format!("{}", d.result.as_ref().unwrap_err());
-            let actual_error = format!("{}", result.unwrap_err());
-            assert!(actual_error == expected_error, msg);
-        }
-    }
-}
-```
-
-## Temporary files
-
-Always delete temporary files on success.
-
-### Golang temporary files
-
-```go
-func TestSomething(t *testing.T) {
-    assert := assert.New(t)
-
-    // Create a temporary directory
-    tmpdir, err := ioutil.TempDir("", "")    
-    assert.NoError(err)             
-
-    // Delete it at the end of the test
-    defer os.RemoveAll(tmpdir) 
-
-    // Add test logic that will use the tmpdir here...
-}
-```
-
-### Rust temporary files
-
-Use the `tempfile` crate which allows files and directories to be deleted
-automatically:
-
-```rust
-#[cfg(test)]
-mod tests {
-    use tempfile::tempdir;
-
-    #[test]
-    fn test_something() {
-
-        // Create a temporary directory (which will be deleted automatically
-        let dir = tempdir().expect("failed to create tmpdir");
-
-        let filename = dir.path().join("file.txt");
-
-        // create filename ...
-    }
-}
-
-```
-
-## Test user
-
-[Unit tests are run *twice*](https://github.com/kata-containers/tests/blob/main/.ci/go-test.sh):
-
-- as the current user
-- as the `root` user (if different to the current user)
-
-When writing a test consider which user should run it; even if the code the
-test is exercising runs as `root`, it may be necessary to *only* run the test
-as a non-`root` for the test to be meaningful. Add appropriate skip
-guards around code that requires `root` and non-`root` so that the test
-will run if the correct type of user is detected and skipped if not.
-
-### Run Golang tests as a different user
-
-The main repository has the most comprehensive set of skip abilities. See:
-
-- https://github.com/kata-containers/kata-containers/tree/main/src/runtime/pkg/katatestutils
-
-### Run Rust tests as a different user
-
-One method is to use the `nix` crate along with some custom macros:
-
-```
-#[cfg(test)]
-mod tests {
-    #[allow(unused_macros)]
-    macro_rules! skip_if_root {
-        () => {
-            if nix::unistd::Uid::effective().is_root() {
-                println!("INFO: skipping {} which needs non-root", module_path!());
-                return;
-            }
-        };
-    }
-
-    #[allow(unused_macros)]
-    macro_rules! skip_if_not_root {
-        () => {
-            if !nix::unistd::Uid::effective().is_root() {
-                println!("INFO: skipping {} which needs root", module_path!());
-                return;
-            }
-        };
-    }
-
-    #[test]
-    fn test_that_must_be_run_as_root() {
-        // Not running as the superuser, so skip.
-        skip_if_not_root!();
-
-        // Run test *iff* the user running the test is root
-
-        // ...
-    }
-}
-```

docs/code-pr-advice.md

@@ -1,246 +0,0 @@
-# Code PR Advice
-
-Before raising a PR containing code changes, we suggest you consider
-the following to ensure a smooth and fast process.
-
-> **Note:**
->
-> - All the advice in this document is optional. However, if the
->   advice provided is not followed, there is no guarantee your PR
->   will be merged.
->
-> - All the check tools will be run automatically on your PR by the CI.
->   However, if you run them locally first, there is a much better
->   chance of a successful initial CI run.
-
-## Assumptions
-
-This document assumes you have already read (and in the case of the
-code of conduct agreed to):
-
-- The [Kata Containers code of conduct](https://github.com/kata-containers/community/blob/main/CODE_OF_CONDUCT.md).
-- The [Kata Containers contributing guide](https://github.com/kata-containers/community/blob/main/CONTRIBUTING.md).
-
-## Code
-
-### Architectures
-
-Do not write architecture-specific code if it is possible to write the
-code generically.
-
-### General advice
-
-- Do not write code to impress: instead write code that is easy to read and understand.
-
-- Always consider which user will run the code. Try to minimise
-  the privileges the code requires.
-
-### Comments
-
-Always add comments if the intent of the code is not obvious. However,
-try to avoid comments if the code could be made clearer (for example
-by using more meaningful variable names).
-
-### Constants
-
-Don't embed magic numbers and strings in functions, particularly if
-they are used repeatedly.
-
-Create constants at the top of the file instead.
-
-### Copyright and license
-
-Ensure all new files contain a copyright statement and an SPDX license
-identifier in the comments at the top of the file.
-
-### FIXME and TODO
-
-If the code contains areas that are not fully implemented, make this
-clear a comment which provides a link to a GitHub issue that provides
-further information.
-
-Do not just rely on comments in this case though: if possible, return
-a "`BUG: feature X not implemented see {bug-url}`" type error.
-
-### Functions
-
-- Keep functions relatively short (less than 100 lines is a good "rule of thumb").
-
-- Document functions if the parameters, return value or general intent
-  of the function is not obvious.
-
-- Always return errors where possible.
-
-  Do not discard error return values from the functions this function
-  calls.
-
-### Logging
-
-- Don't use multiple log calls when a single log call could be used.
-
-- Use structured logging where possible to allow
-  [standard tooling](https://github.com/kata-containers/tests/tree/main/cmd/log-parser)
-  be able to extract the log fields.
-
-### Names
-
-Give functions, macros and variables clear and meaningful names.
-
-### Structures
-
-#### Golang structures
-
-Unlike Rust, Go does not enforce that all structure members be set.
-This has lead to numerous bugs in the past where code like the
-following is used:
-
-```go
-type Foo struct {
-    Key   string
-    Value string
-}
-
-// BUG: Key not set, but nobody noticed! ;(
-let foo1 = Foo {
-    Value: "foo",
-}
-```
-
-A much safer approach is to create a constructor function to enforce
-integrity:
-
-```go
-type Foo struct {
-    Key   string
-    Value string
-}
-
-func NewFoo(key, value string) (*Foo, error) {
-    if key == "" {
-        return nil, errors.New("Foo needs a key")
-    }
-
-    if value == "" {
-        return nil, errors.New("Foo needs a value")
-    }
-
-    return &Foo{
-        Key:   key,
-        Value: value,
-    }, nil
-}
-
-func testFoo() error {
-    // BUG: Key not set, but nobody noticed! ;(
-    badFoo := Foo{Value: "value"}
-
-    // Ok - the constructor performs needed validation
-    goodFoo, err := NewFoo("name", "value")
-    if err != nil {
-        return err
-    }
-
-    return nil
-```
-
-> **Note:**
->
-> The above is just an example. The *safest* approach would be to move
-> `NewFoo()` into a separate package and make `Foo` and it's elements
-> private. The compiler would then enforce the use of the constructor
-> to guarantee correctly defined objects.
-
-
-### Tracing
-
-Consider if the code needs to create a new
-[trace span](https://github.com/kata-containers/kata-containers/blob/main/docs/tracing.md).
-
-Ensure any new trace spans added to the code are completed.
-
-## Tests
-
-### Unit tests
-
-Where possible, code changes should be accompanied by unit tests.
-
-Consider using the standard
-[table-based approach](https://github.com/kata-containers/tests/blob/main/Unit-Test-Advice.md)
-as it encourages you to make functions small and simple, and also
-allows you to think about what types of value to test.
-
-### Other categories of test
-
-Raised a GitHub issue in the
-[`tests`](https://github.com/kata-containers/tests) repository that
-explains what sort of test is required along with as much detail as
-possible. Ensure the original issue is referenced on the `tests` issue.
-
-### Unsafe code
-
-#### Rust language specifics
-
-Minimise the use of `unsafe` blocks in Rust code and since it is
-potentially dangerous always write [unit tests][#unit-tests]
-for this code where possible.
-
-`expect()` and `unwrap()` will cause the code to panic on error.
-Prefer to return a `Result` on error rather than using these calls to
-allow the caller to deal with the error condition.
-
-The table below lists the small number of cases where use of
-`expect()` and `unwrap()` are permitted:
-
-| Area | Rationale for permitting |
-|-|-|
-| In test code (the `tests` module) | Panics will cause the test to fail, which is desirable. |
-| `lazy_static!()` | This magic macro cannot "return" a value as it runs before `main()`. |
-| `defer!()` | Similar to golang's `defer()` but doesn't allow the use of `?`. |
-| `tokio::spawn(async move {})` | Cannot currently return a `Result` from an `async move` closure. |
-| If an explicit test is performed before the `unwrap()` / `expect()` | *"Just about acceptable"*, but not ideal `[*]` |
-
-
-`[*]` - There can lead to bad *future* code: consider what would
-happen if the explicit test gets dropped in the future. This is easier
-to happen if the test and the extraction of the value are two separate
-operations. In summary, this strategy can introduce an insidious
-maintenance issue.
-
-## Documentation
-
-### General requirements
-
-- All new features should be accompanied by documentation explaining:
-
-  - What the new feature does
-
-  - Why it is useful
-
-  - How to use the feature
-
-  - Any known issues or limitations
-
-    Links should be provided to GitHub issues tracking the issues
-
-- The [documentation requirements document](Documentation-Requirements.md)
-  explains how the project formats documentation.
-
-### Markdown syntax
-
-Run the
-[markdown checker](https://github.com/kata-containers/tests/tree/main/cmd/check-markdown)
-on your documentation changes.
-
-### Spell check
-
-Run the
-[spell checker](https://github.com/kata-containers/tests/tree/main/cmd/check-spelling)
-on your documentation changes.
-
-## Finally
-
-You may wish to read the documentation that the
-[Kata Review Team](https://github.com/kata-containers/community/blob/main/Rota-Process.md) use to help review PRs:
-
-- [PR review guide](https://github.com/kata-containers/community/blob/main/PR-Review-Guide.md).
-- [documentation review process](https://github.com/kata-containers/community/blob/main/Documentation-Review-Process.md).

docs/design/arch-images/kata-2-metrics.drawio

@@ -1 +1 @@
-<mxfile host="app.diagrams.net" modified="2021-11-05T13:07:32.992Z" agent="5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36" etag="j5e7J3AOXxeQrt-Zz2uw" version="15.6.8" type="device"><diagram id="XNV8G0dePIPkhS_Khqr4" name="Page-1">7Vxdd9o4EP01nLP7QI5s+fORUNhNT7rNbnqaZl/2CCywG2OxQhDIr18Z29iyZD6CDZRuHho8toQ9986dGVlNC3Yny98omvqfiIfDlg68ZQt+aOm6AUyb/4otq8TiGnpiGNPAS0wgNzwGbzgxapl1Hnh4ltoSEyMkZMFUNA5JFOEhE2yIUvIqXjYioScYpmiMJcPjEIWy9SnwmJ9YdQjd/MTvOBj76VdDCNI7n6Ds6tQw85FHXgsm2GvBLiWEJZ8myy4OY++JjulXnN3cGcUR22fAn3fPzx+jj7e9HrIXA330feIZ7czPCxTO00dO75atMh+MKZlP08swZXip8jwaZJcD+ca0zeNyomAywYyu+CXpRG3NNM1kUMoSXU+PX3OfG9nEfsHdG56gFOfxZvbcE/xD6gy1Yz7/88nuPiLwcNcZDLvEvZ10Vm1zt1+4WyIPx5NoLXj76gcMP07RMD77yqOB23w2CdPTIxKxlN78nuHtmCIv4A7qkpDQ9XzQxsjC8blREIYFu4ewMxpy+4xR8oILZ6yhgwcjfkZ2+Xa0yzDK2JzP81ZzntcNtedHI8+1LNnzo9FIHyo971kDy7Qa8Xx61gJCSGhyRHCtkXHRLLMhXGwJF659/KFrCwtHBkAbIA3rKgAAsHqdfjqDCBn/aRIYTRORUWiVa8rAwKZwcSRcuCQzFESYcrNWLz6K4HFtD9i2QrZM7HiGCjtHH8Ak3ETs+n0A+v0msdNhCTv7RkZPM1TwNSV37lb4asw6VwifDc4MnqJ88ldTTBfBjHulDB1/SibiI/o2IhEuAZGaUBiMI3445B7lvIC3sc8CXqZ20hOTwPPir1ESIqcMUORDn9DgLaZcmF7QnHKWUhqQ4TNUpUZj6GkSeuM5nvGUBl4wjeJW5odAsDnAzBJihiLgrIYgUz6DrJYSRqdvVwzblol82qJZM3Y75sfrV9wKGC+pXdEa7BTP16/s7/lLbVc0uY+8hn7lYGCkdgVKyJy0XdHkPvJn6VcOxu4C2xVte7t5zf3K0fCdv12Ry6efpl05XDgvrFvR5V7zmruVw/E6Z7OiDjcoQYK9MX5MDwllPhmTCIW93FpyXn7NPSHTFMXvmLFV6lI0Z0TEGC8D9q3w+TmeiueN5OjDMp15fbCSMdJijDg0dPUtuzI+KMwSH+bTrI+yeWRss3xO5nSYsfb+y53jDp6+P1r+y+fXj+HLU97AMETHmG1xalo+xI7cygqKQ8SCBRZuQwXxemiHUrQqXDAlQcRmhZkfYoPQBNqOmJstu0gYxQjD1ksjnBLFkrvICbd5nCNUA0qqwRidDpXMvEcDLiMCm/ZXAopnwVvaV8dcSF3IJzdvW+YHBctktmwNo727+erWHdxormWIMpEcHUYXGV0osmFz09kUZDSaYSZJymEIqyNHLqirEb5A7Xmv1hTZZB+nPa6sPVtFaqf45HyDBrRFvtnHEa5WQqklQy7ir5g6aiE6hjpqEbuQtOUCUf52p63yCFOzS6w7Lm1t9WtB1LqFNrMXjfmnlm6FcYE74CZrHH/6ZdOLeq04F/T5v92/7tqff5UoXeeyj+U5tqXsPWHHNGA2w37LPtlLib0L37auOa4IKpQXpDXHkktfq4bSV4lf1tb+HBpyXOmr75t+4L7pZ28ROQpjXY7RBxrP7eP5LH5wTBdYXla4rsATyz4LqALPaCbw1Mn7nHGXx9pzMdR2xF0eas/ZfCfJ3VDRcqrFrPa4e2fytk1bSbfq5F0eYTpmrclbyUH5XaTP2FRJzMvsOPUKJTi44QQ3Um6uqd/MXm9l/aZuiFM01x7ICwqV6J5MdqCY8QGwTqI98aQPmAbcpTFTj1wC21+P4J56tGGhCQEUK8QjaVhNs8NVzbHFezNAaSP7TlUrjTha1dDX0f1d+292B77+8dRGX7/MfHCmzPpOplaycPcah9tIslM1lpq4jWZTPGWTJBGTjjuGYVXftKXpLY0wHbh9hA5ca9uIZtpkKKfaF8RQe0KigCle6V3sXofDa2/NNfWbCgIVqm/dVLxgba76lrcvXGjb+64yetvK1u4VMKtuYTkOKjl0frQXI5VR8446FZqmXlNlCsqvQkqyXktpqnxnvMdWvOZ3hzqGmAgMR7EmYG928hR1yW5s05XCMcnaqRcsBAdZ/87j/5C4pmR7tuZkh1+gGdPlmpjZ+WzFNV9wbc/8YNJep5/FZnp+t+tvSC6uLx8ZDeejrfQ6aDtqBdTNrbzKujYjw5f6XK/a8esAwIt4hes7JgAGOKXrs/2o2o1e2qUtb51T1QZ1bL5SPoL8mvYCxEn5pkDNWKcpxir3rv+vToeGiF1BnMtSJ3n16ArUaX/XV6qTKW9Wq0md+GH+RwaSSiv/Ww2w9x8=</diagram></mxfile>
+<mxfile host="Chrome" modified="2020-07-02T06:44:28.736Z" agent="5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" etag="r7FpfnbGNK7jbg54Gu9x" version="13.3.5" type="device"><diagram id="XNV8G0dePIPkhS_Khqr4" name="Page-1">7VvZcuI4FP0aHqFky+sjkNCTqfR0qtLV6fTLlMDy0hiLscWWrx8Zy3iRQkhjQxbygnVlhK1zz9HRkg4cztZfYjT3vxIHhx0VOOsOvOqoKrRthX2kkU0WMTWYBbw4cLIQKAL3wRPOgkoeXQQOTngsC1FCQhrMq8EJiSI8oZUYimOyqt7mktCpBObIw0LgfoJCMfoQONTPoiqEdlHxFw48n/80hIA/+Qzld/NA4iOHrEoheN2Bw5gQml3N1kMcpr1X7ZjRM7W7J4txRA/5wrd/v5rDewTubvrjyZDYg1l/01W0rJklChf8lfnT0k3eBzFZRA5OW1E6cLDyA4rv52iS1q4Y6izm01nIq10SUQ4jwxAOvBg5AXvCIQlJvG0PmhgZOK1zgzAsxR2ELXfC4gmNyRSXaoyJhccuqxHfmXfDEscUr0sh3gdfMJlhGm/YLby2q+i6nn2J56QOzKy8KhDWctT8Eri7rEQ8q7xd60W/swve9a+BQW8PBlWTw+C6jm0YIgyu66oTKQyOMTZ0oykYNLsGQ/7OJRgYnUQYFENvCYYWUXgvZFDss5PBuHBBVc7OBQUKvY4dNjbyIompTzwSofC6iA4KXNKULu65JWTO0fiNKd1wONCCkipWeB3Qn6Xrx7Spns5LV2ve8raw4YUyy7R9iCRkEU/4u3i3328se/zw+97wp99Wf4fTh2I0pCj2MN3TOZwjaYfsBTjGIaLBsmomGodKhQJjKI3nEymAt2jMPFql01EYeBG7nrAOwyy/B2nqBswE9XnFLHCcDF+cBE9ovG0v7fo5CSK6fR190NGvDgJjb7YJpNlZO/6rFfMkJRPoKbahVUUtKx2MBm/8Ln27Ustqmonldrt2tQ3iugnLmzqeu4c8COK9mVkRRSNkXTpwgmUFZeO/RWopt0h0ky0UfXaDos3XWzzyenblpZ+sfykKIhw73cQPZt0poqi73DXPHnf7C9nNzY2Hmqi2yhgpWJWpLQDGdX/EW6jqM/trSoUts6bCUBwLFdPMk6Csw0YDg6EcePMV3H6D4szwiDc/y4XSt9Ji8bVtSSbq5nGibouivpdjL6o6TxjQA5ZuVjMGHKc0jQqJfKwQzdQHzpwj7YAkc+Sj17nswN7HLklGofHNCbglCrjhWKahyQQc9nUN5i20JeBMnMHLAi6bzLQm35r+megGjqKbeqhQw0OF+jR8U0W+3cVp2z5eJOmL45gl8ccmnm1XiGdIltQUS2uHePJx7py8K7j2WKbaC7wrqPaYt3eSYQ5KZr1yMTsb76QQi1Min9K5FPe3OelVnyHaq+e8oMfYVWXgkVPefIKrKL3aAlN71lRcxXgWz5PxWP2YRIYHEnmXXxBa1fSyj8uvRrMJ/XBvb7q/6A348c9DF/34nvjgzANA61lzgizJMX4jNguKer9dqpqRKKCkXX917pUpW1drS48yh6Xqp1yZ0kS9Tshk2hwOsl0xCwATynDo6wBooG0cLFibYFoiCjIQYGs2VxH6+43OL/9omNu32vLyqozxpuyqKurXe9uleZYyf+BYoa6rx3mInJWGUuFkVwOn86yKgOllW6Zp0TVr2zKaRHRPvS2jiWT+8IOfqcBeDQodqucd/8TdMeSl7/iRzaCm1bYpVREEWwZCW0dFLAGEnXilCtcsGJLTO7bpANOUEEbHliNdFbXUMczO+1SBGo0AGI2aAgqqdaC0XKTK0qWdkjB79oZYWJw0f1qsDMkgc1Kk8vN15fWwzRzHyyCRzHbZi9IqGNWOjEiEa73OQ4f7Shn61blE/aidT+LgKc2vsPPCssWrzizWBVB2ZlF2ZLE1qEQb6C1wkprUKY6j9Ez8u4CrmeEJVNGBsk1Y46TwiCdKP59L0HOhOpdLkJxkutgE2dCjw/PbBGW/p7v4hB1Y5tl9gmjpLj5B6hNka+Yn9QmqaOkuPmGHjtaeT2DF4h/tsrW/4v8V4fX/</diagram></mxfile>

docs/design/data/metrics.yaml

@@ -1825,8 +1825,12 @@ components:
                   desc: ""
                 - value: grpc.StartContainerRequest
                   desc: ""
+                - value: grpc.StartTracingRequest
+                  desc: ""
                 - value: grpc.StatsContainerRequest
                   desc: ""
+                - value: grpc.StopTracingRequest
+                  desc: ""
                 - value: grpc.TtyWinResizeRequest
                   desc: ""
                 - value: grpc.UpdateContainerRequest

docs/design/host-cgroups.md

@@ -242,8 +242,8 @@ On the other hand, running all non vCPU threads under a dedicated overhead cgrou
 accurate metrics on the actual Kata Container pod overhead, allowing for tuning the overhead
 cgroup size and constraints accordingly.
 
-[linux-config]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md
-[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/master/config-linux.md#cgroups-path
+[linux-config]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md
+[cgroupspath]: https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#cgroups-path
 
 # Supported cgroups
 

docs/design/kata-2-0-metrics.md

@@ -1,21 +1,21 @@
 # Kata 2.0 Metrics Design
 
-Kata implements CRI's API and supports [`ContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L101) and [`ListContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L103) interfaces to expose containers metrics. User can use these interfaces to get basic metrics about containers.
+Kata implement CRI's API and support [`ContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L101) and [`ListContainerStats`](https://github.com/kubernetes/kubernetes/blob/release-1.18/staging/src/k8s.io/cri-api/pkg/apis/runtime/v1alpha2/api.proto#L103) interfaces to expose containers metrics. User can use these interface to get basic metrics about container.
 
-Unlike `runc`, Kata is a VM-based runtime and has a different architecture.
+But unlike `runc`, Kata is a VM-based runtime and has a different architecture.
 
-## Limitations of Kata 1.x and target of Kata 2.0
+## Limitations of Kata 1.x and the target of Kata 2.0
 
 Kata 1.x has a number of limitations related to observability that may be obstacles to running Kata Containers at scale.
 
-In Kata 2.0, the following components will be able to provide more details about the system:
+In Kata 2.0, the following components will be able to provide more details about the system.
 
 - containerd shim v2 (effectively `kata-runtime`)
 - Hypervisor statistics
 - Agent process
 - Guest OS statistics
 
-> **Note**: In Kata 1.x, the main user-facing component was the runtime (`kata-runtime`). From 1.5, Kata introduced the Kata containerd shim v2 (`containerd-shim-kata-v2`) which is essentially a modified runtime that is loaded by containerd to simplify and improve the way VM-based containers are created and managed.
+> **Note**: In Kata 1.x, the main user-facing component was the runtime (`kata-runtime`). From 1.5, Kata then introduced the Kata containerd shim v2 (`containerd-shim-kata-v2`) which is essentially a modified runtime that is loaded by containerd to simplify and improve the way VM-based containers are created and managed.
 >
 > For Kata 2.0, the main component is the Kata containerd shim v2, although the deprecated `kata-runtime` binary will be maintained for a period of time.
 >
@@ -25,15 +25,14 @@ In Kata 2.0, the following components will be able to provide more details about
 
 Kata 2.0 metrics strongly depend on [Prometheus](https://prometheus.io/), a graduated project from CNCF.
 
-Kata Containers 2.0 introduces a new Kata component called `kata-monitor` which is used to monitor the Kata components on the host. It's shipped with the Kata runtime to provide an interface to:
+Kata Containers 2.0 introduces a new Kata component called `kata-monitor` which is used to monitor the other Kata components on the host. It's the monitor interface with Kata runtime, and we can do something like these:
 
 - Get metrics
 - Get events
 
-At present, `kata-monitor` supports retrieval of metrics only: this is what will be covered in this document.
+In this document we will cover metrics only. And until now it only supports metrics function.
 
-
-This is the architecture overview of metrics in Kata Containers 2.0:
+This is the architecture overview metrics in Kata Containers 2.0.
 
 ![Kata Containers 2.0 metrics](arch-images/kata-2-metrics.png)
 
@@ -46,38 +45,38 @@ For a quick evaluation, you can check out [this how to](../how-to/how-to-set-pro
 
 ### Kata monitor
 
-The `kata-monitor` management agent should be started on each node where the Kata containers runtime is installed. `kata-monitor` will:
+`kata-monitor` is a management agent on one node, where many Kata containers are running. `kata-monitor`'s work include:
 
-> **Note**: a *node* running Kata containers will be either a single host system or a worker node belonging to a K8s cluster capable of running Kata pods.
+> **Note**: node is a single host system or a node in K8s clusters.
 
-- Aggregate sandbox metrics running on the node, adding the `sandbox_id` label to them.
-- Expose a new Prometheus target, allowing all node metrics coming from the Kata shim to be collected by Prometheus indirectly. This simplifies the targets count in Prometheus and avoids exposing shim's metrics by `ip:port`.
+- Aggregate sandbox metrics running on this node, and add `sandbox_id` label
+- As a Prometheus target, all metrics from Kata shim on this node will be collected by Prometheus indirectly. This can easy the targets count in Prometheus, and also need not to expose shim's metrics by `ip:port`
 
-Only one `kata-monitor` process runs in each node.
+Only one `kata-monitor` process are running on one node.
 
-`kata-monitor` uses a different communication channel than the one used by the container engine (`containerd`/`CRI-O`) to communicate with the Kata shim. The Kata shim exposes a dedicated socket address reserved to `kata-monitor`.
+`kata-monitor` is using a different communication channel other than that `conatinerd` communicating with Kata shim, and Kata shim listen on a new socket address for communicating with `kata-monitor`.
 
-The shim's metrics socket file is created under the virtcontainers sandboxes directory, i.e. `vc/sbs/${PODID}/shim-monitor.sock`.
+The way `kata-monitor` get shim's metrics socket file(`monitor_address`) like that `containerd` get shim address. The socket is an abstract socket and saved as file `abstract` with the same directory of `address` for `containerd`.
 
-> **Note**: If there is no Prometheus server configured, i.e., there are no scrape operations, `kata-monitor` will not collect any metrics.
+> **Note**: If there is no Prometheus server is configured, i.e., there is no scrape operations, `kata-monitor` will do nothing initiative.
 
 ### Kata runtime
 
-Kata runtime is responsible for:
+Runtime is responsible for:
 
 - Gather metrics about shim process
 - Gather metrics about hypervisor process
 - Gather metrics about running sandbox
-- Get metrics from Kata agent (through `ttrpc`)
+- Get metrics from Kata agent(through `ttrpc`)
 
 ### Kata agent
 
-Kata agent is responsible for:
+Agent is responsible for:
 
 - Gather agent process metrics
 - Gather guest OS metrics
 
-In Kata 2.0, the agent adds a new interface:
+And in Kata 2.0, agent will add a new interface:
 
 ```protobuf
 rpc GetMetrics(GetMetricsRequest) returns (Metrics);
@@ -94,49 +93,33 @@ The `metrics` field is Prometheus encoded content. This can avoid defining a fix
 
 ### Performance and overhead
 
-Metrics should not become a bottleneck for the system or downgrade the performance: they should run with minimal overhead.
+Metrics should not become the bottleneck of system, downgrade the performance, and run with minimal overhead.
 
 Requirements:
 
 * Metrics **MUST** be quick to collect
-* Metrics **MUST** be small
+* Metrics **MUST** be small.
 * Metrics **MUST** be generated only if there are subscribers to the Kata metrics service
 * Metrics **MUST** be stateless
 
-In Kata 2.0, metrics are collected only when needed (pull mode), mainly from the `/proc` filesystem, and consumed by Prometheus. This means that if the Prometheus collector is not running (so no one cares about the metrics) the overhead will be zero.
+In Kata 2.0, metrics are collected mainly from `/proc` filesystem, and consumed by Prometheus, based on a pull mode, that is mean if there is no Prometheus collector is running, so there will be zero overhead if nobody cares the metrics.
 
-The metrics service also doesn't hold any metrics in memory.
-
-#### Metrics size ####
+Metrics service also doesn't hold any metrics in memory.
 
 |\*|No Sandbox | 1 Sandbox | 2 Sandboxes |
 |---|---|---|---|
 |Metrics count| 39 | 106 | 173 |
-|Metrics size (bytes)| 9K | 144K | 283K |
-|Metrics size (`gzipped`, bytes)| 2K | 10K | 17K |
+|Metrics size(bytes)| 9K | 144K | 283K |
+|Metrics size(`gzipped`, bytes)| 2K | 10K | 17K |
 
-*Metrics size*: response size of one Prometheus scrape request.
+*Metrics size*: Response size of one Prometheus scrape request.
 
-It's easy to estimate the size of one metrics fetch request issued by Prometheus.
-The formula to calculate the expected size when no gzip compression is in place is:  
-9 + (144 - 9) * `number of kata sandboxes`
-
-Prometheus supports `gzip compression`. When enabled, the response size of each request will be smaller:  
-2 + (10 - 2) * `number of kata sandboxes`
-
-**Example**  
-We have 10 sandboxes running on a node. The expected size of one metrics fetch request issued by Prometheus against the kata-monitor agent running on that node will be:  
-9 + (144 - 9) * 10 = **1.35M**
-
-If `gzip compression` is enabled:  
-2 + (10 - 2) * 10 = **82K**
-
-#### Metrics delay ####
+It's easy to estimated that if there are 10 sandboxes running in the host, the size of one metrics fetch request issued by Prometheus will be about to 9 + (144 - 9) * 10 = 1.35M (not `gzipped`) or 2 + (10 - 2) * 10 = 82K (`gzipped`). Of course Prometheus support `gzip` compression, that can reduce the response size of every request.
 
 And here is some test data:
 
-- End-to-end (from Prometheus server to `kata-monitor` and `kata-monitor` write response back): **20ms**(avg)
-- Agent (RPC all from shim to agent): **3ms**(avg)
+- End-to-end (from Prometheus server to `kata-monitor` and `kata-monitor` write response back): 20ms(avg)
+- Agent(RPC all from shim to agent): 3ms(avg)
 
 Test infrastructure:
 
@@ -145,13 +128,13 @@ Test infrastructure:
 
 **Scrape interval**
 
-Prometheus default `scrape_interval` is 1 minute, but it is usually set to 15 seconds. A smaller `scrape_interval` causes more overhead, so users should set it depending on their monitoring needs.
+Prometheus default `scrape_interval` is 1 minute, and usually it is set to 15s. Small `scrape_interval` will cause more overhead, so user should set it on monitor demand.
 
 ## Metrics list
 
-Here are listed all the metrics supported by Kata 2.0. Some metrics are dependent on the VM guest kernel, so the available ones may differ based on the environment.
+Here listed is all supported metrics by Kata 2.0. Some metrics is dependent on guest kernels in the VM, so there may be some different by your environment.
 
-Metrics are categorized by the component from/for which the metrics are collected.
+Metrics is categorized by component where metrics are collected from and for.
 
 * [Metric types](#metric-types)
 * [Kata agent metrics](#kata-agent-metrics)
@@ -162,15 +145,15 @@ Metrics are categorized by the component from/for which the metrics are collecte
 * [Kata containerd shim v2 metrics](#kata-containerd-shim-v2-metrics)
 
 > **Note**:
->  * Labels here do not include the `instance` and `job` labels added by Prometheus.
+>  * Labels here are not include `instance` and `job` labels that added by Prometheus.
 >  * Notes about metrics unit
 >    * `Kibibytes`, abbreviated `KiB`. 1 `KiB` equals 1024 B.
->    * For some metrics (like network devices statistics from file `/proc/net/dev`), unit depends on label( for example `recv_bytes` and `recv_packets` have different units).
->    * Most of these metrics are collected from the `/proc` filesystem, so the unit of each metric matches the unit of the relevant `/proc` entry. See the `proc(5)` manual page for further details.
+>    * For some metrics (like network devices statistics from file `/proc/net/dev`), unit is depend on label( for example `recv_bytes` and `recv_packets` are having different units).
+>    * Most of these metrics is collected from `/proc` filesystem, so the unit of metrics are keeping the same unit as `/proc`. See the `proc(5)` manual page for further details.
 
 ### Metric types
 
-Prometheus offers four core metric types.
+Prometheus offer four core metric types.
 
 - Counter: A counter is a cumulative metric that represents a single monotonically increasing counter whose value can only increase.
 
@@ -305,7 +288,7 @@ Metrics about Kata containerd shim v2 process.
 
 | Metric name | Type | Units | Labels | Introduced in Kata version |
 |---|---|---|---|---|
-| `kata_shim_agent_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (RPC actions of Kata agent)<ul><li>`grpc.CheckRequest`</li><li>`grpc.CloseStdinRequest`</li><li>`grpc.CopyFileRequest`</li><li>`grpc.CreateContainerRequest`</li><li>`grpc.CreateSandboxRequest`</li><li>`grpc.DestroySandboxRequest`</li><li>`grpc.ExecProcessRequest`</li><li>`grpc.GetMetricsRequest`</li><li>`grpc.GuestDetailsRequest`</li><li>`grpc.ListInterfacesRequest`</li><li>`grpc.ListProcessesRequest`</li><li>`grpc.ListRoutesRequest`</li><li>`grpc.MemHotplugByProbeRequest`</li><li>`grpc.OnlineCPUMemRequest`</li><li>`grpc.PauseContainerRequest`</li><li>`grpc.RemoveContainerRequest`</li><li>`grpc.ReseedRandomDevRequest`</li><li>`grpc.ResumeContainerRequest`</li><li>`grpc.SetGuestDateTimeRequest`</li><li>`grpc.SignalProcessRequest`</li><li>`grpc.StartContainerRequest`</li><li>`grpc.StatsContainerRequest`</li><li>`grpc.TtyWinResizeRequest`</li><li>`grpc.UpdateContainerRequest`</li><li>`grpc.UpdateInterfaceRequest`</li><li>`grpc.UpdateRoutesRequest`</li><li>`grpc.WaitProcessRequest`</li><li>`grpc.WriteStreamRequest`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
+| `kata_shim_agent_rpc_durations_histogram_milliseconds`: <br> RPC latency distributions. | `HISTOGRAM` | `milliseconds` | <ul><li>`action` (RPC actions of Kata agent)<ul><li>`grpc.CheckRequest`</li><li>`grpc.CloseStdinRequest`</li><li>`grpc.CopyFileRequest`</li><li>`grpc.CreateContainerRequest`</li><li>`grpc.CreateSandboxRequest`</li><li>`grpc.DestroySandboxRequest`</li><li>`grpc.ExecProcessRequest`</li><li>`grpc.GetMetricsRequest`</li><li>`grpc.GuestDetailsRequest`</li><li>`grpc.ListInterfacesRequest`</li><li>`grpc.ListProcessesRequest`</li><li>`grpc.ListRoutesRequest`</li><li>`grpc.MemHotplugByProbeRequest`</li><li>`grpc.OnlineCPUMemRequest`</li><li>`grpc.PauseContainerRequest`</li><li>`grpc.RemoveContainerRequest`</li><li>`grpc.ReseedRandomDevRequest`</li><li>`grpc.ResumeContainerRequest`</li><li>`grpc.SetGuestDateTimeRequest`</li><li>`grpc.SignalProcessRequest`</li><li>`grpc.StartContainerRequest`</li><li>`grpc.StartTracingRequest`</li><li>`grpc.StatsContainerRequest`</li><li>`grpc.StopTracingRequest`</li><li>`grpc.TtyWinResizeRequest`</li><li>`grpc.UpdateContainerRequest`</li><li>`grpc.UpdateInterfaceRequest`</li><li>`grpc.UpdateRoutesRequest`</li><li>`grpc.WaitProcessRequest`</li><li>`grpc.WriteStreamRequest`</li></ul></li><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_shim_fds`: <br> Kata containerd shim v2 open FDs. | `GAUGE` |  | <ul><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_shim_go_gc_duration_seconds`: <br> A summary of the pause duration of garbage collection cycles. | `SUMMARY` | `seconds` | <ul><li>`sandbox_id`</li></ul> | 2.0.0 |
 | `kata_shim_go_goroutines`: <br> Number of goroutines that currently exist. | `GAUGE` |  | <ul><li>`sandbox_id`</li></ul> | 2.0.0 |

docs/how-to/run-kata-with-k8s.md

@@ -22,7 +22,7 @@ An equivalent shim implementation for CRI-O is planned.
 ### CRI-O
 For CRI-O installation instructions, refer to the [CRI-O Tutorial](https://github.com/cri-o/cri-o/blob/main/tutorial.md) page.
 
-The following sections show how to set up the CRI-O snippet configuration file (default path: `/etc/crio/crio.conf`) for Kata.
+The following sections show how to set up the CRI-O configuration file (default path: `/etc/crio/crio.conf`) for Kata.
 
 Unless otherwise stated, all the following settings are specific to the `crio.runtime` table:
 ```toml
@@ -40,16 +40,74 @@ A comprehensive documentation of the configuration file can be found [here](http
 #### Kubernetes Runtime Class (CRI-O v1.12+)
 The [Kubernetes Runtime Class](https://kubernetes.io/docs/concepts/containers/runtime-class/)
 is the preferred way of specifying the container runtime configuration to run a Pod's containers.
-To use this feature, Kata must added as a runtime handler. This can be done by
-dropping a `50-kata` snippet file into `/etc/crio/crio.conf.d`, with the
-content shown below:
+To use this feature, Kata must added as a runtime handler with:
 
 ```toml
-[crio.runtime.runtimes.kata]
-	runtime_path = "/usr/bin/containerd-shim-kata-v2"
-	runtime_type = "vm"
-	runtime_root = "/run/vc"
-	privileged_without_host_devices = true
+[crio.runtime.runtimes.kata-runtime]
+  runtime_path = "/usr/bin/kata-runtime"
+  runtime_type = "oci"
+```
+
+You can also add multiple entries to specify alternatives hypervisors, e.g.:
+```toml
+[crio.runtime.runtimes.kata-qemu]
+  runtime_path = "/usr/bin/kata-runtime"
+  runtime_type = "oci"
+
+[crio.runtime.runtimes.kata-fc]
+  runtime_path = "/usr/bin/kata-runtime"
+  runtime_type = "oci"
+```
+
+#### Untrusted annotation (until CRI-O v1.12)
+The untrusted annotation is used to specify a runtime for __untrusted__ workloads, i.e.
+a runtime to be used when the workload cannot be trusted and a higher level of security
+is required. An additional flag can be used to let CRI-O know if a workload
+should be considered _trusted_ or _untrusted_ by default.
+For further details, see the documentation
+[here](../design/architecture.md#mixing-vm-based-and-namespace-based-runtimes).
+
+```toml
+# runtime is the OCI compatible runtime used for trusted container workloads.
+# This is a mandatory setting as this runtime will be the default one
+# and will also be used for untrusted container workloads if
+# runtime_untrusted_workload is not set.
+runtime = "/usr/bin/runc"
+
+# runtime_untrusted_workload is the OCI compatible runtime used for untrusted
+# container workloads. This is an optional setting, except if
+# default_container_trust is set to "untrusted".
+runtime_untrusted_workload = "/usr/bin/kata-runtime"
+
+# default_workload_trust is the default level of trust crio puts in container
+# workloads. It can either be "trusted" or "untrusted", and the default
+# is "trusted".
+# Containers can be run through different container runtimes, depending on
+# the trust hints we receive from kubelet:
+# - If kubelet tags a container workload as untrusted, crio will try first to
+# run it through the untrusted container workload runtime. If it is not set,
+# crio will use the trusted runtime.
+# - If kubelet does not provide any information about the container workload trust
+# level, the selected runtime will depend on the default_container_trust setting.
+# If it is set to "untrusted", then all containers except for the host privileged
+# ones, will be run by the runtime_untrusted_workload runtime. Host privileged
+# containers are by definition trusted and will always use the trusted container
+# runtime. If default_container_trust is set to "trusted", crio will use the trusted
+# container runtime for all containers.
+default_workload_trust = "untrusted"
+```
+
+#### Network namespace management
+To enable networking for the workloads run by Kata, CRI-O needs to be configured to
+manage network namespaces, by setting the following key to `true`.
+
+In CRI-O v1.16:
+```toml
+manage_network_ns_lifecycle = true
+```
+In CRI-O v1.17+:
+```toml
+manage_ns_lifecycle = true
 ```
 
 

docs/install/README.md

@@ -12,26 +12,16 @@ Containers.
 
 Packaged installation methods uses your distribution's native package format (such as RPM or DEB).
 
-> **Note:** We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are
-> easily applied.
+*Note:* We encourage installation methods that provides automatic updates, it ensures security updates and bug fixes are
+easily applied.
 
-| Installation method                                  | Description                                                                                  | Automatic updates | Use case                                                                                      |
-|------------------------------------------------------|----------------------------------------------------------------------------------------------|-------------------|-----------------------------------------------------------------------------------------------|
-| [Using kata-deploy](#kata-deploy-installation)       | The preferred way to deploy the Kata Containers distributed binaries on a Kubernetes cluster | **No!**           | Best way to give it a try on kata-containers on an already up and running Kubernetes cluster. | 
-| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories                          | yes               | Recommended for most users.                                                                   |
-| [Using snap](#snap-installation)                     | Easy to install                                                                              | yes               | Good alternative to official distro packages.                                                 |
-| [Automatic](#automatic-installation)                 | Run a single command to install a full system                                                | **No!**           | For those wanting the latest release quickly.                                                 |
-| [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system                                      | **No!**           | For those who want the latest release with more control.                                      |
-| [Build from source](#build-from-source-installation) | Build the software components manually                                                       | **No!**           | Power users and developers only.                                                              |
-
-### Kata Deploy Installation
-
-Kata Deploy provides a Dockerfile, which contains all of the binaries and
-artifacts required to run Kata Containers, as well as reference DaemonSets,
-which can be utilized to install Kata Containers on a running Kubernetes
-cluster.
-
-[Use Kata Deploy](/tools/packaging/kata-deploy/README.md) to install Kata Containers on a Kubernetes Cluster.
+| Installation method                                  | Description                                                         | Automatic updates | Use case                                                 |
+|------------------------------------------------------|---------------------------------------------------------------------|-------------------|----------------------------------------------------------|
+| [Using official distro packages](#official-packages) | Kata packages provided by Linux distributions official repositories | yes               | Recommended for most users.                              |
+| [Using snap](#snap-installation)                     | Easy to install                                                     | yes               | Good alternative to official distro packages.            |
+| [Automatic](#automatic-installation)                 | Run a single command to install a full system                       | **No!**           | For those wanting the latest release quickly.            |
+| [Manual](#manual-installation)                       | Follow a guide step-by-step to install a working system             | **No!**           | For those who want the latest release with more control. |
+| [Build from source](#build-from-source-installation) | Build the software components manually                              | **No!**           | Power users and developers only.                         |
 
 ### Official packages
 
@@ -58,9 +48,9 @@ Follow the [containerd installation guide](container-manager/containerd/containe
 
 ## Build from source installation
 
-> **Note:** Power users who decide to build from sources should be aware of the
-> implications of using an unpackaged system which will not be automatically
-> updated as new [releases](../Stable-Branch-Strategy.md) are made available.
+*Note:* Power users who decide to build from sources should be aware of the
+implications of using an unpackaged system which will not be automatically
+updated as new [releases](../Stable-Branch-Strategy.md) are made available.
 
 [Building from sources](../Developer-Guide.md#initial-setup)  allows power users
 who are comfortable building software from source to use the latest component

docs/tracing.md

@@ -203,12 +203,11 @@ is highly recommended. For working with the agent, you may also wish to
 [enable a debug console][setup-debug-console]
 to allow you to access the VM environment.
 
-[agent-ctl]: https://github.com/kata-containers/kata-containers/blob/main/tools/agent-ctl
 [enable-full-debug]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#enable-full-debug
 [jaeger-all-in-one]: https://www.jaegertracing.io/docs/getting-started/
 [jaeger-tracing]: https://www.jaegertracing.io
 [opentelemetry]: https://opentelemetry.io
 [osbuilder]: https://github.com/kata-containers/kata-containers/blob/main/tools/osbuilder
 [setup-debug-console]: https://github.com/kata-containers/kata-containers/blob/main/docs/Developer-Guide.md#set-up-a-debug-console
-[trace-forwarder]: https://github.com/kata-containers/kata-containers/blob/main/src/trace-forwarder
+[trace-forwarder]: /src/trace-forwarder
 [vsock]: https://wiki.qemu.org/Features/VirtioVsock

docs/use-cases/using-SPDK-vhostuser-and-kata.md

@@ -1,4 +1,4 @@
-# Setup to run SPDK vhost-user devices with Kata Containers
+# Setup to run SPDK vhost-user devices with Kata Containers and Docker*
 
 > **Note:** This guide only applies to QEMU, since the vhost-user storage
 > device is only available for QEMU now. The enablement work on other
@@ -222,43 +222,26 @@ minor `0` should be created for it, in order to be recognized by Kata runtime:
 $ sudo mknod /var/run/kata-containers/vhost-user/block/devices/vhostblk0 b 241 0
 ```
 
+> **Note:** The enablement of vhost-user block device in Kata containers
+> is supported by Kata Containers `1.11.0-alpha1` or newer.
+> Make sure you have updated your Kata containers before evaluation.
+
 ## Launch a Kata container with SPDK vhost-user block device
 
-To use `vhost-user-blk` device, use `ctr` to pass a host `vhost-user-blk`
-device to the container. In your `config.json`, you should use `devices`
+To use `vhost-user-blk` device, use Docker to pass a host `vhost-user-blk`
+device to the container. In docker, `--device=HOST-DIR:CONTAINER-DIR` is used
 to pass a host device to the container.
 
-For example (only `vhost-user-blk` listed):
-
-```json
-{
-  "linux": {
-    "devices": [
-      {
-        "path": "/dev/vda",
-        "type": "b",
-        "major": 241,
-        "minor": 0,
-        "fileMode": 420,
-        "uid": 0,
-        "gid": 0
-      }
-    ]
-  }
-}
-```
-
-With `rootfs` provisioned under `bundle` directory, you can run your SPDK container:
+For example:
 
 ```bash
-$ sudo ctr run -d --runtime io.containerd.run.kata.v2 --config bundle/config.json spdk_container
+$ sudo docker run --runtime kata-runtime --device=/var/run/kata-containers/vhost-user/block/devices/vhostblk0:/dev/vda -it busybox sh
 ```
 
 Example of performing I/O operations on the `vhost-user-blk` device inside
 container:
 
 ```
-$ sudo ctr t exec --exec-id 1 -t spdk_container sh
 / # ls -l /dev/vda
 brw-r--r--    1 root     root      254,   0 Jan 20 03:54 /dev/vda
 / # dd if=/dev/vda of=/tmp/ddtest bs=4k count=20

pkg/logging/Cargo.toml

@@ -7,15 +7,15 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde_json = "1.0.39"
+serde_json = "1.0.73"
 # slog:
 # - Dynamic keys required to allow HashMap keys to be slog::Serialized.
 # - The 'max_*' features allow changing the log level at runtime
 #   (by stopping the compiler from removing log calls).
-slog = { version = "2.5.2", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
-slog-json = "2.3.0"
-slog-async = "2.3.0"
-slog-scope = "4.1.2"
+slog = { version = "2.7.0", features = ["dynamic-keys", "max_level_trace", "release_max_level_debug"] }
+slog-json = "2.4.0"
+slog-async = "2.7.0"
+slog-scope = "4.4.0"
 
 [dev-dependencies]
-tempfile = "3.1.0"
+tempfile = "3.2.0"

pkg/logging/Makefile

@@ -1,18 +0,0 @@
-# Copyright (c) 2021 Intel Corporation
-#
-# SPDX-License-Identifier: Apache-2.0
-#
-
-# It is not necessary to have a build target as this crate is built
-# automatically by the consumers of it.
-#
-# However, it is essential that the crate be tested.
-default: test
-
-# It is essential to run these tests using *both* build profiles.
-# See the `test_logger_levels()` test for further information.
-test:
-	@echo "INFO: testing log levels for development build"
-	@cargo test
-	@echo "INFO: testing log levels for release build"
-	@cargo test --release

pkg/logging/src/lib.rs

@@ -20,8 +20,6 @@ const LOG_LEVELS: &[(&str, slog::Level)] = &[
     ("critical", slog::Level::Critical),
 ];
 
-const DEFAULT_SUBSYSTEM: &str = "root";
-
 // XXX: 'writer' param used to make testing possible.
 pub fn create_logger<W>(
     name: &str,
@@ -52,7 +50,7 @@ where
     let logger = slog::Logger::root(
         async_drain.fuse(),
         o!("version" => env!("CARGO_PKG_VERSION"),
-            "subsystem" => DEFAULT_SUBSYSTEM,
+            "subsystem" => "root",
             "pid" => process::id().to_string(),
             "name" => name.to_string(),
             "source" => source.to_string()),
@@ -218,8 +216,8 @@ where
 #[cfg(test)]
 mod tests {
     use super::*;
-    use serde_json::{json, Value};
-    use slog::{crit, debug, error, info, warn, Logger};
+    use serde_json::Value;
+    use slog::info;
     use std::io::prelude::*;
     use tempfile::NamedTempFile;
 
@@ -297,15 +295,15 @@ mod tests {
                 let result_level = result.unwrap();
                 let expected_level = d.result.unwrap();
 
-                assert!(result_level == expected_level, "{}", msg);
+                assert!(result_level == expected_level, msg);
                 continue;
             } else {
-                assert!(result.is_err(), "{}", msg);
+                assert!(result.is_err(), msg);
             }
 
-            let expected_error = d.result.as_ref().unwrap_err();
-            let actual_error = result.unwrap_err();
-            assert!(&actual_error == expected_error, "{}", msg);
+            let expected_error = format!("{}", d.result.as_ref().unwrap_err());
+            let actual_error = format!("{}", result.unwrap_err());
+            assert!(actual_error == expected_error, msg);
         }
     }
 
@@ -352,13 +350,13 @@ mod tests {
             let msg = format!("{}, result: {:?}", msg, result);
 
             if d.result.is_ok() {
-                assert!(result == d.result, "{}", msg);
+                assert!(result == d.result, msg);
                 continue;
             }
 
-            let expected_error = d.result.as_ref().unwrap_err();
-            let actual_error = result.unwrap_err();
-            assert!(&actual_error == expected_error, "{}", msg);
+            let expected_error = format!("{}", d.result.as_ref().unwrap_err());
+            let actual_error = format!("{}", result.unwrap_err());
+            assert!(actual_error == expected_error, msg);
         }
     }
 
@@ -378,17 +376,14 @@ mod tests {
         let record_key = "record-key-1";
         let record_value = "record-key-2";
 
-        let (logger, guard) = create_logger(name, source, level, writer);
+        let logger = create_logger(name, source, level, writer);
 
         let msg = "foo, bar, baz";
 
         // Call the logger (which calls the drain)
-        // Note: This "mid level" log level should be available in debug or
-        // release builds.
-        info!(&logger, "{}", msg; "subsystem" => record_subsystem, record_key => record_value);
+        info!(logger, "{}", msg; "subsystem" => record_subsystem, record_key => record_value);
 
         // Force temp file to be flushed
-        drop(guard);
         drop(logger);
 
         let mut contents = String::new();
@@ -435,168 +430,4 @@ mod tests {
             .expect("failed to find record key field");
         assert_eq!(field_record_value, record_value);
     }
-
-    #[test]
-    fn test_logger_levels() {
-        let name = "name";
-        let source = "source";
-
-        let debug_msg = "a debug log level message";
-        let info_msg = "an info log level message";
-        let warn_msg = "a warn log level message";
-        let error_msg = "an error log level message";
-        let critical_msg = "a critical log level message";
-
-        // The slog crate will *remove* macro calls for log levels "above" the
-        // configured log level.lock
-        //
-        // At the time of writing, the default slog log
-        // level is "info", but this crate overrides that using the magic
-        // "*max_level*" features in the "Cargo.toml" manifest.
-
-        // However, there are two log levels:
-        //
-        // - max_level_${level}
-        //
-        //   This is the log level for normal "cargo build" (development/debug)
-        //   builds.
-        //
-        // - release_max_level_${level}
-        //
-        //   This is the log level for "cargo install" and
-        //   "cargo build --release" (release) builds.
-        //
-        // This crate sets them to different values, which is sensible and
-        // standard practice. However, that causes a problem: there is
-        // currently no clean way for this test code to detect _which_
-        // profile the test is being built for (development or release),
-        // meaning we cannot know which macros are expected to produce output
-        // and which aren't ;(
-        //
-        // The best we can do is test the following log levels which
-        // are expected to work in all build profiles.
-
-        let debug_closure = |logger: &Logger, msg: String| debug!(logger, "{}", msg);
-        let info_closure = |logger: &Logger, msg: String| info!(logger, "{}", msg);
-        let warn_closure = |logger: &Logger, msg: String| warn!(logger, "{}", msg);
-        let error_closure = |logger: &Logger, msg: String| error!(logger, "{}", msg);
-        let critical_closure = |logger: &Logger, msg: String| crit!(logger, "{}", msg);
-
-        struct TestData<'a> {
-            slog_level: slog::Level,
-            slog_level_tag: &'a str,
-            msg: String,
-            closure: Box<dyn Fn(&Logger, String)>,
-        }
-
-        let tests = &[
-            TestData {
-                slog_level: slog::Level::Debug,
-                // Looks like a typo but tragically it isn't! ;(
-                slog_level_tag: "DEBG",
-                msg: debug_msg.into(),
-                closure: Box::new(debug_closure),
-            },
-            TestData {
-                slog_level: slog::Level::Info,
-                slog_level_tag: "INFO",
-                msg: info_msg.into(),
-                closure: Box::new(info_closure),
-            },
-            TestData {
-                slog_level: slog::Level::Warning,
-                slog_level_tag: "WARN",
-                msg: warn_msg.into(),
-                closure: Box::new(warn_closure),
-            },
-            TestData {
-                slog_level: slog::Level::Error,
-                // Another language tragedy
-                slog_level_tag: "ERRO",
-                msg: error_msg.into(),
-                closure: Box::new(error_closure),
-            },
-            TestData {
-                slog_level: slog::Level::Critical,
-                slog_level_tag: "CRIT",
-                msg: critical_msg.into(),
-                closure: Box::new(critical_closure),
-            },
-        ];
-
-        for (i, d) in tests.iter().enumerate() {
-            let msg = format!("test[{}]", i);
-
-            // Create a writer for the logger drain to use
-            let writer =
-                NamedTempFile::new().expect(&format!("{:}: failed to create tempfile", msg));
-
-            // Used to check file contents before the temp file is unlinked
-            let mut writer_ref = writer
-                .reopen()
-                .expect(&format!("{:?}: failed to clone tempfile", msg));
-
-            let (logger, logger_guard) = create_logger(name, source, d.slog_level, writer);
-
-            // Call the logger (which calls the drain)
-            (d.closure)(&logger, d.msg.to_owned());
-
-            // Force temp file to be flushed
-            drop(logger_guard);
-            drop(logger);
-
-            let mut contents = String::new();
-            writer_ref
-                .read_to_string(&mut contents)
-                .expect(&format!("{:?}: failed to read tempfile contents", msg));
-
-            // Convert file to JSON
-            let fields: Value = serde_json::from_str(&contents)
-                .expect(&format!("{:?}: failed to convert logfile to json", msg));
-
-            // Check the expected JSON fields
-
-            let field_ts = fields
-                .get("ts")
-                .expect(&format!("{:?}: failed to find timestamp field", msg));
-            assert_ne!(field_ts, "", "{}", msg);
-
-            let field_version = fields
-                .get("version")
-                .expect(&format!("{:?}: failed to find version field", msg));
-            assert_eq!(field_version, env!("CARGO_PKG_VERSION"), "{}", msg);
-
-            let field_pid = fields
-                .get("pid")
-                .expect(&format!("{:?}: failed to find pid field", msg));
-            assert_ne!(field_pid, "", "{}", msg);
-
-            let field_level = fields
-                .get("level")
-                .expect(&format!("{:?}: failed to find level field", msg));
-            assert_eq!(field_level, d.slog_level_tag, "{}", msg);
-
-            let field_msg = fields
-                .get("msg")
-                .expect(&format!("{:?}: failed to find msg field", msg));
-            assert_eq!(field_msg, &json!(d.msg), "{}", msg);
-
-            let field_name = fields
-                .get("name")
-                .expect(&format!("{:?}: failed to find name field", msg));
-            assert_eq!(field_name, name, "{}", msg);
-
-            let field_source = fields
-                .get("source")
-                .expect(&format!("{:?}: failed to find source field", msg));
-            assert_eq!(field_source, source, "{}", msg);
-
-            let field_subsystem = fields
-                .get("subsystem")
-                .expect(&format!("{:?}: failed to find subsystem field", msg));
-
-            // No explicit subsystem, so should be the default
-            assert_eq!(field_subsystem, &json!(DEFAULT_SUBSYSTEM), "{}", msg);
-        }
-    }
 }

src/agent/Cargo.toml

@@ -24,7 +24,7 @@ serial_test = "0.5.1"
 # Async helpers
 async-trait = "0.1.42"
 async-recursion = "0.3.2"
-futures = "0.3.12"
+futures = "0.3.17"
 
 # Async runtime
 tokio = { version = "1", features = ["full"] }
@@ -45,10 +45,10 @@ slog-scope = "4.1.2"
 slog-stdlog = "4.0.0"
 log = "0.4.11"
 
-prometheus = { version = "0.9.0", features = ["process"] }
-procfs = "0.7.9"
+prometheus = { version = "0.13.0", features = ["process"] }
+procfs = "0.12.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 
 # Tracing
 tracing = "0.1.26"

src/agent/Makefile

@@ -101,10 +101,7 @@ endef
 ##TARGET default: build code
 default: $(TARGET) show-header
 
-$(TARGET): $(GENERATED_CODE) logging-crate-tests $(TARGET_PATH)
-
-logging-crate-tests:
-	make -C $(CWD)/../../pkg/logging
+$(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): $(SOURCES) | show-summary
 	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
@@ -114,7 +111,7 @@ $(GENERATED_FILES): %: %.in
 
 ##TARGET optimize: optimized  build
 optimize: $(SOURCES) | show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny-warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
 
 ##TARGET clippy: run clippy linter
 clippy: $(GENERATED_CODE)
@@ -208,10 +205,9 @@ codecov-html: check_tarpaulin
 
 .PHONY: \
 	help \
-	logging-crate-tests \
-	optimize \
 	show-header \
 	show-summary \
+	optimize \
 	vendor
 
 ##TARGET generate-protocols: generate/update grpc agent protocols

src/agent/README.md

@@ -1,38 +1,48 @@
-# Kata Agent
+# Kata Agent in Rust
 
-## Overview
+This is a rust version of the [`kata-agent`](https://github.com/kata-containers/agent).
 
-The Kata agent is a long running process that runs inside the Virtual Machine
-(VM) (also known as the "pod" or "sandbox").
+In Denver PTG, [we discussed about re-writing agent in rust](https://etherpad.openstack.org/p/katacontainers-2019-ptg-denver-agenda):
 
-The agent is packaged inside the Kata Containers
-[guest image](../../docs/design/architecture.md#guest-image)
-which is used to boot the VM. Once the runtime has launched the configured
-[hypervisor](../../docs/hypervisors.md) to create a new VM, the agent is
-started. From this point on, the agent is responsible for creating and
-managing the life cycle of the containers inside the VM.
+> In general, we all think about re-write agent in rust to reduce the footprint of agent. Moreover, Eric mentioned the possibility to stop using gRPC, which may have some impact on footprint. We may begin to do some POC to show how much we could save by re-writing agent in rust.
 
-For further details, see the
-[architecture document](../../docs/design/architecture.md).
+After that, we drafted the initial code here, and any contributions are welcome.
 
-## Audience
+## Features
 
-If you simply wish to use Kata Containers, it is not necessary to understand
-the details of how the agent operates. Please see the
-[installation documentation](../../docs/install) for details of how deploy
-Kata Containers (which will include the Kata agent).
+| Feature | Status |
+| :--|:--:|
+| **OCI Behaviors** |
+| create/start containers | :white_check_mark: |
+| signal/wait process     | :white_check_mark: |
+| exec/list process       | :white_check_mark: |
+| I/O stream              | :white_check_mark: |
+| Cgroups                 | :white_check_mark: |
+| Capabilities, `rlimit`, readonly path, masked path, users | :white_check_mark: |
+| Seccomp                 | :white_check_mark: |
+| container stats (`stats_container`)                     | :white_check_mark: |
+| Hooks                   | :white_check_mark: |
+| **Agent Features & APIs** |
+| run agent as `init` (mount fs, udev, setup `lo`) | :white_check_mark: |
+| block device as root device                      | :white_check_mark: |
+| Health API                                       | :white_check_mark: |
+| network, interface/routes (`update_container`)   | :white_check_mark: |
+| File transfer API (`copy_file`)                  | :white_check_mark: |
+| Device APIs (`reseed_random_device`, , `online_cpu_memory`, `mem_hotplug_probe`, `set_guet_data_time`) | :white_check_mark: |
+| VSOCK support                                    | :white_check_mark: |
+| virtio-serial support                            | :heavy_multiplication_x: |
+| OCI Spec validator                               | :white_check_mark: |
+| **Infrastructures**|
+| Debug Console | :white_check_mark: |
+| Command line  | :white_check_mark: |
+| Tracing       | :heavy_multiplication_x: |
 
-The remainder of this document is only useful for developers and testers.
+## Getting Started
 
-## Build from Source
+### Build from Source
+The rust-agent needs to be built statically and linked with `musl`
 
-Since the agent is written in the Rust language this section assumes the tool
-chain has been installed using standard Rust `rustup` tool.
-
-### Build with musl
-
-If you wish to build the agent with the `musl` C library, you need to run the
-following commands:
+> **Note:** skip this step for ppc64le, the build scripts explicitly use gnu for ppc64le.
 
 ```bash
 $ arch=$(uname -m)
@@ -40,15 +50,12 @@ $ rustup target add "${arch}-unknown-linux-musl"
 $ sudo ln -s /usr/bin/g++ /bin/musl-g++
 ```
 
-> **Note:**
->
-> It is not currently possible to build using `musl` on ppc64le and s390x
-> since both platforms lack the `musl` target.
-
-### Build the agent binary
-
-The following steps download the Kata Containers source files and build the agent:
+ppc64le-only: Manually install `protoc`, e.g.
+```bash
+$ sudo dnf install protobuf-compiler
+```
 
+Download the source files in the Kata containers repository and build the agent:
 ```bash
 $ GOPATH="${GOPATH:-$HOME/go}"
 $ dir="$GOPATH/src/github.com/kata-containers"
@@ -56,56 +63,17 @@ $ git -C ${dir} clone --depth 1 https://github.com/kata-containers/kata-containe
 $ make -C ${dir}/kata-containers/src/agent
 ```
 
-## Change the agent API
-
-The Kata runtime communicates with the Kata agent using a ttRPC based API protocol.
-
-This ttRPC API is defined by a set of [protocol buffers files](protocols/protos).
-The protocol files are used to generate the bindings for the following components:
-
-| Component | Language | Generation method | Tooling required |
-|-|-|-|-|
-| runtime | Golang | Run, `make generate-protocols` | `protoc` |
-| agent | Rust | Run, `make` |  |
-
-If you wish to change the API, these files must be regenerated. Although the
-rust code will be automatically generated by the
-[build script](protocols/build.rs),
-the Golang code generation requires the external `protoc` command to be
-available in `$PATH`.
-
-To install the `protoc` command on a Fedora/CentOS/RHEL system:
+## Run Kata CI with rust-agent
+   * Firstly, install Kata as noted by ["how to install Kata"](../../docs/install/README.md)
+   * Secondly, build your own Kata initrd/image following the steps in ["how to build your own initrd/image"](../../docs/Developer-Guide.md#create-and-install-rootfs-and-initrd-image).
+notes: Please use your rust agent instead of the go agent when building your initrd/image.
+   * Clone the Kata CI test cases from: https://github.com/kata-containers/tests.git, and then run the CRI test with: 
 
 ```bash
-$ sudo dnf install -y protobuf-compiler
+$sudo -E PATH=$PATH -E GOPATH=$GOPATH integration/containerd/shimv2/shimv2-tests.sh
 ```
 
-## Custom guest image and kernel assets
-
-If you wish to develop or test changes to the agent, you will need to create a
-custom guest image using the [osbuilder tool](../../tools/osbuilder). You
-may also wish to create a custom [guest kernel](../../tools/packaging/kernel).
-
-Once created, [configure](../runtime/README.md#configuration) Kata Containers to use
-these custom assets to allow you to test your changes.
-
-> **Note:**
->
-> To simplify development and testing, you may wish to run the agent
-> [stand alone](#run-the-agent-stand-alone) initially.
-
-## Tracing
-
-For details of tracing the operation of the agent, see the
-[tracing documentation](../../docs/tracing.md).
-
-## Run the agent stand alone
-
-Although the agent is designed to run in a VM environment, for development and
-testing purposes it is possible to run it as a normal application.
-
-When run in this way, the agent can be controlled using the low-level Kata
-agent control tool, rather than the Kata runtime.
-
-For further details, see the
-[agent control tool documentation](../../tools/agent-ctl/README.md#run-the-tool-and-the-agent-in-the-same-environment).
+## Mini Benchmark
+The memory of `RssAnon` consumed by the go-agent and rust-agent as below:
+go-agent: about 11M
+rust-agent: about 1.1M

src/agent/oci/Cargo.toml

@@ -5,7 +5,7 @@ authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
 
 [dependencies]
-serde = "1.0.91"
-serde_derive = "1.0.91"
-serde_json = "1.0.39"
-libc = "0.2.58"
+serde = "1.0.131"
+serde_derive = "1.0.131"
+serde_json = "1.0.73"
+libc = "0.2.112"

src/agent/protocols/Cargo.toml

@@ -4,16 +4,10 @@ version = "0.1.0"
 authors = ["The Kata Containers community <kata-dev@lists.katacontainers.io>"]
 edition = "2018"
 
-[features]
-default = []
-with-serde = [ "serde", "serde_json" ]
-
 [dependencies]
 ttrpc = { version = "0.5.0", features = ["async"] }
 async-trait = "0.1.42"
-protobuf = { version = "=2.14.0", features = ["with-serde"] }
-serde = { version = "1.0.130", features = ["derive"], optional = true }
-serde_json = { version = "1.0.68", optional = true }
+protobuf = "=2.14.0"
 
 [build-dependencies]
 ttrpc-codegen = "0.2.0"

src/agent/protocols/build.rs

@@ -3,148 +3,29 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use std::fs::File;
-use std::io::{BufRead, BufReader, Read, Write};
-use std::path::Path;
-use std::process::exit;
-use ttrpc_codegen::{Codegen, Customize, ProtobufCustomize};
-
-fn replace_text_in_file(file_name: &str, from: &str, to: &str) -> Result<(), std::io::Error> {
-    let mut src = File::open(file_name)?;
-    let mut contents = String::new();
-    src.read_to_string(&mut contents).unwrap();
-    drop(src);
-
-    let new_contents = contents.replace(from, to);
-
-    let mut dst = File::create(&file_name)?;
-    dst.write_all(new_contents.as_bytes())?;
-
-    Ok(())
-}
-
-fn use_serde(protos: &[&str], out_dir: &Path) -> Result<(), std::io::Error> {
-    protos
-        .iter()
-        .try_for_each(|f: &&str| -> Result<(), std::io::Error> {
-            let out_file = Path::new(f)
-                .file_name()
-                .and_then(|s| s.to_str())
-                .ok_or(format!("failed to get proto file name for {:?}", f))
-                .map(|s| {
-                    let t = s.replace(".proto", ".rs");
-                    out_dir.join(t)
-                })
-                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?
-                .to_str()
-                .ok_or(format!("cannot convert {:?} path to string", f))
-                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?
-                .to_string();
-
-            replace_text_in_file(
-                &out_file,
-                "derive(Serialize, Deserialize)",
-                "derive(serde::Serialize, serde::Deserialize)",
-            )
-        })
-}
-
-fn handle_file(autogen_comment: &str, rust_filename: &str) -> Result<(), std::io::Error> {
-    let mut new_contents = Vec::new();
-
-    let file = File::open(rust_filename)?;
-
-    let reader = BufReader::new(file);
-
-    // Guard the code since it is only needed for the agent-ctl tool,
-    // not the agent itself.
-    let serde_default_code = r#"#[cfg_attr(feature = "with-serde", serde(default))]"#;
-
-    for line in reader.lines() {
-        let line = line?;
-
-        new_contents.push(line.clone());
-
-        let pattern = "//! Generated file from";
-
-        if line.starts_with(&pattern) {
-            new_contents.push(autogen_comment.into());
-        }
-
-        let struct_pattern = "pub struct ";
-
-        // Although we've requested serde support via `Customize`, to
-        // allow the `kata-agent-ctl` tool to partially deserialise structures
-        // specified in JSON, we need this bit of additional magic.
-        if line.starts_with(&struct_pattern) {
-            new_contents.insert(new_contents.len() - 1, serde_default_code.trim().into());
-        }
-    }
-
-    let data = new_contents.join("\n");
-
-    let mut dst = File::create(&rust_filename)?;
-
-    dst.write_all(data.as_bytes())?;
-
-    Ok(())
-}
-
-fn real_main() -> Result<(), std::io::Error> {
-    let autogen_comment = format!("\n//! Generated by {:?} ({:?})", file!(), module_path!());
+use std::fs;
+use ttrpc_codegen::{Codegen, Customize};
 
+fn main() {
     let protos = vec![
-        "protos/agent.proto",
-        "protos/google/protobuf/empty.proto",
-        "protos/health.proto",
-        "protos/oci.proto",
         "protos/types.proto",
+        "protos/agent.proto",
+        "protos/health.proto",
+        "protos/google/protobuf/empty.proto",
+        "protos/oci.proto",
     ];
 
-    // Tell Cargo that if the .proto files changed, to rerun this build script.
-    protos
-        .iter()
-        .for_each(|p| println!("cargo:rerun-if-changed={}", &p));
-
-    let ttrpc_options = Customize {
-        async_server: true,
-        ..Default::default()
-    };
-
-    let protobuf_options = ProtobufCustomize {
-        serde_derive: Some(true),
-        ..Default::default()
-    };
-
-    let out_dir = Path::new("src");
-
     Codegen::new()
-        .out_dir(out_dir)
+        .out_dir("src")
         .inputs(&protos)
         .include("protos")
-        .customize(ttrpc_options)
         .rust_protobuf()
-        .rust_protobuf_customize(protobuf_options)
-        .run()?;
-
-    for file in protos.iter() {
-        let proto_filename = Path::new(file).file_name().unwrap();
-
-        let generated_file = proto_filename
-            .to_str()
-            .ok_or("failed")
-            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?
-            .replace(".proto", ".rs");
-
-        let out_file = out_dir.join(generated_file);
-
-        let out_file_str = out_file
-            .to_str()
-            .ok_or("failed")
-            .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))?;
-
-        handle_file(&autogen_comment, out_file_str)?;
-    }
+        .customize(Customize {
+            async_server: true,
+            ..Default::default()
+        })
+        .run()
+        .expect("Gen codes failed.");
 
     // There is a message named 'Box' in oci.proto
     // so there is a struct named 'Box', we should replace Box<Self> to ::std::boxed::Box<Self>
@@ -153,16 +34,11 @@ fn real_main() -> Result<(), std::io::Error> {
         "src/oci.rs",
         "self: Box<Self>",
         "self: ::std::boxed::Box<Self>",
-    )?;
-
-    use_serde(&protos, out_dir)?;
-
-    Ok(())
+    )
+    .unwrap();
 }
 
-fn main() {
-    if let Err(e) = real_main() {
-        eprintln!("ERROR: {}", e);
-        exit(1);
-    }
+fn replace_text_in_file(file_name: &str, from: &str, to: &str) -> Result<(), std::io::Error> {
+    let new_contents = fs::read_to_string(file_name)?.replace(from, to);
+    fs::write(&file_name, new_contents.as_bytes())
 }

src/agent/protocols/protos/agent.proto

@@ -52,6 +52,8 @@ service AgentService {
 	rpc AddARPNeighbors(AddARPNeighborsRequest) returns (google.protobuf.Empty);
 
 	// observability
+	rpc StartTracing(StartTracingRequest) returns (google.protobuf.Empty);
+	rpc StopTracing(StopTracingRequest) returns (google.protobuf.Empty);
 	rpc GetMetrics(GetMetricsRequest) returns (Metrics);
 
 	// misc (TODO: some rpcs can be replaced by hyperstart-exec)
@@ -490,6 +492,12 @@ message CopyFileRequest {
 	bytes data = 8;
 }
 
+message StartTracingRequest {
+}
+
+message StopTracingRequest {
+}
+
 message GetOOMEventRequest {}
 
 message OOMEvent {

src/agent/rustjail/Cargo.toml

@@ -23,7 +23,7 @@ scan_fmt = "0.2"
 regex = "1.1"
 path-absolutize = "1.2.0"
 anyhow = "1.0.32"
-cgroups = { package = "cgroups-rs", version = "0.2.5" }
+cgroups = { package = "cgroups-rs", version = "0.2.8" }
 rlimit = "0.5.3"
 
 tokio = { version = "1.2.0", features = ["sync", "io-util", "process", "time", "macros"] }

src/agent/rustjail/src/container.rs

@@ -636,10 +636,11 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
 
     // setup the envs
     for e in env.iter() {
-        match valid_env(e) {
-            Some((key, value)) => env::set_var(key, value),
-            None => log_child!(cfd_log, "invalid env key-value: {:?}", e),
+        let v: Vec<&str> = e.splitn(2, '=').collect();
+        if v.len() != 2 {
+            continue;
         }
+        env::set_var(v[0], v[1]);
     }
 
     // set the "HOME" env getting from "/etc/passwd", if
@@ -663,8 +664,8 @@ fn do_init_child(cwfd: RawFd) -> Result<()> {
     let _ = unistd::close(crfd);
     let _ = unistd::close(cwfd);
 
-    unistd::setsid().context("create a new session")?;
     if oci_process.terminal {
+        unistd::setsid()?;
         unsafe {
             libc::ioctl(0, libc::TIOCSCTTY);
         }
@@ -994,6 +995,8 @@ impl BaseContainer for LinuxContainer {
 
         info!(logger, "entered namespaces!");
 
+        self.created = SystemTime::now();
+
         if p.init {
             let spec = self.config.spec.as_mut().unwrap();
             update_namespaces(&self.logger, spec, p.pid)?;
@@ -1562,30 +1565,6 @@ async fn execute_hook(logger: &Logger, h: &Hook, st: &OCIState) -> Result<()> {
     }
 }
 
-// valid environment variables according to https://doc.rust-lang.org/std/env/fn.set_var.html#panics
-fn valid_env(e: &str) -> Option<(&str, &str)> {
-    // wherther key or value will contain NULL char.
-    if e.as_bytes().contains(&b'\0') {
-        return None;
-    }
-
-    let v: Vec<&str> = e.splitn(2, '=').collect();
-
-    // key can't hold an `equal` sign, but value can
-    if v.len() != 2 {
-        return None;
-    }
-
-    let (key, value) = (v[0].trim(), v[1].trim());
-
-    // key can't be empty
-    if key.is_empty() {
-        return None;
-    }
-
-    Some((key, value))
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -2009,49 +1988,4 @@ mod tests {
         let ret = do_init_child(std::io::stdin().as_raw_fd());
         assert!(ret.is_err(), "Expecting Err, Got {:?}", ret);
     }
-
-    #[test]
-    fn test_valid_env() {
-        let env = valid_env("a=b=c");
-        assert_eq!(Some(("a", "b=c")), env);
-
-        let env = valid_env("a=b");
-        assert_eq!(Some(("a", "b")), env);
-        let env = valid_env("a =b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env(" a =b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env("a= b");
-        assert_eq!(Some(("a", "b")), env);
-
-        let env = valid_env("a=b ");
-        assert_eq!(Some(("a", "b")), env);
-        let env = valid_env("a=b c ");
-        assert_eq!(Some(("a", "b c")), env);
-
-        let env = valid_env("=b");
-        assert_eq!(None, env);
-
-        let env = valid_env("a=");
-        assert_eq!(Some(("a", "")), env);
-
-        let env = valid_env("a==");
-        assert_eq!(Some(("a", "=")), env);
-
-        let env = valid_env("a");
-        assert_eq!(None, env);
-
-        let invalid_str = vec![97, b'\0', 98];
-        let invalid_string = std::str::from_utf8(&invalid_str).unwrap();
-
-        let invalid_env = format!("{}=value", invalid_string);
-        let env = valid_env(&invalid_env);
-        assert_eq!(None, env);
-
-        let invalid_env = format!("key={}", invalid_string);
-        let env = valid_env(&invalid_env);
-        assert_eq!(None, env);
-    }
 }

src/agent/rustjail/src/mount.rs

@@ -112,7 +112,6 @@ lazy_static! {
 }
 
 #[inline(always)]
-#[cfg(not(test))]
 pub fn mount<
     P1: ?Sized + NixPath,
     P2: ?Sized + NixPath,
@@ -125,42 +124,21 @@ pub fn mount<
     flags: MsFlags,
     data: Option<&P4>,
 ) -> std::result::Result<(), nix::Error> {
-    mount::mount(source, target, fstype, flags, data)
+    #[cfg(not(test))]
+    return mount::mount(source, target, fstype, flags, data);
+    #[cfg(test)]
+    return Ok(());
 }
 
 #[inline(always)]
-#[cfg(test)]
-pub fn mount<
-    P1: ?Sized + NixPath,
-    P2: ?Sized + NixPath,
-    P3: ?Sized + NixPath,
-    P4: ?Sized + NixPath,
->(
-    _source: Option<&P1>,
-    _target: &P2,
-    _fstype: Option<&P3>,
-    _flags: MsFlags,
-    _data: Option<&P4>,
-) -> std::result::Result<(), nix::Error> {
-    Ok(())
-}
-
-#[inline(always)]
-#[cfg(not(test))]
 pub fn umount2<P: ?Sized + NixPath>(
     target: &P,
     flags: MntFlags,
 ) -> std::result::Result<(), nix::Error> {
-    mount::umount2(target, flags)
-}
-
-#[inline(always)]
-#[cfg(test)]
-pub fn umount2<P: ?Sized + NixPath>(
-    _target: &P,
-    _flags: MntFlags,
-) -> std::result::Result<(), nix::Error> {
-    Ok(())
+    #[cfg(not(test))]
+    return mount::umount2(target, flags);
+    #[cfg(test)]
+    return Ok(());
 }
 
 pub fn init_rootfs(
@@ -472,20 +450,14 @@ fn mount_cgroups(
     Ok(())
 }
 
-#[cfg(not(test))]
 fn pivot_root<P1: ?Sized + NixPath, P2: ?Sized + NixPath>(
     new_root: &P1,
     put_old: &P2,
 ) -> anyhow::Result<(), nix::Error> {
-    unistd::pivot_root(new_root, put_old)
-}
-
-#[cfg(test)]
-fn pivot_root<P1: ?Sized + NixPath, P2: ?Sized + NixPath>(
-    _new_root: &P1,
-    _put_old: &P2,
-) -> anyhow::Result<(), nix::Error> {
-    Ok(())
+    #[cfg(not(test))]
+    return unistd::pivot_root(new_root, put_old);
+    #[cfg(test)]
+    return Ok(());
 }
 
 pub fn pivot_rootfs<P: ?Sized + NixPath + std::fmt::Debug>(path: &P) -> Result<()> {
@@ -610,15 +582,11 @@ fn parse_mount_table() -> Result<Vec<Info>> {
 }
 
 #[inline(always)]
-#[cfg(not(test))]
 fn chroot<P: ?Sized + NixPath>(path: &P) -> Result<(), nix::Error> {
-    unistd::chroot(path)
-}
-
-#[inline(always)]
-#[cfg(test)]
-fn chroot<P: ?Sized + NixPath>(_path: &P) -> Result<(), nix::Error> {
-    Ok(())
+    #[cfg(not(test))]
+    return unistd::chroot(path);
+    #[cfg(test)]
+    return Ok(());
 }
 
 pub fn ms_move_root(rootfs: &str) -> Result<bool> {
@@ -777,7 +745,7 @@ fn mount_from(
         let _ = fs::create_dir_all(&dir).map_err(|e| {
             log_child!(
                 cfd_log,
-                "creat dir {}: {}",
+                "create dir {}: {}",
                 dir.to_str().unwrap(),
                 e.to_string()
             )

src/agent/samples/configuration-all-endpoints.toml

@@ -29,7 +29,9 @@ allowed = [
         "SetGuestDateTimeRequest",
         "SignalProcessRequest",
         "StartContainerRequest",
+        "StartTracingRequest",
         "StatsContainerRequest",
+        "StopTracingRequest",
         "TtyWinResizeRequest",
         "UpdateContainerRequest",
         "UpdateInterfaceRequest",

src/agent/src/device.rs

@@ -3,6 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
+use libc::{c_uint, major, minor};
 use nix::sys::stat;
 use regex::Regex;
 use std::collections::HashMap;
@@ -11,7 +12,7 @@ use std::fmt;
 use std::fs;
 use std::os::unix::ffi::OsStrExt;
 use std::os::unix::fs::MetadataExt;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
 use tokio::sync::Mutex;
@@ -22,7 +23,7 @@ use crate::linux_abi::*;
 use crate::pci;
 use crate::sandbox::Sandbox;
 use crate::uevent::{wait_for_uevent, Uevent, UeventMatcher};
-use anyhow::{anyhow, Context, Result};
+use anyhow::{anyhow, Result};
 use oci::{LinuxDeviceCgroup, LinuxResources, Spec};
 use protocols::agent::Device;
 use tracing::instrument;
@@ -52,6 +53,15 @@ pub const DRIVER_VFIO_GK_TYPE: &str = "vfio-gk";
 // container as a VFIO device node
 pub const DRIVER_VFIO_TYPE: &str = "vfio";
 
+#[derive(Debug)]
+struct DevIndexEntry {
+    idx: usize,
+    residx: Vec<usize>,
+}
+
+#[derive(Debug)]
+struct DevIndex(HashMap<String, DevIndexEntry>);
+
 #[instrument]
 pub fn online_device(path: &str) -> Result<()> {
     fs::write(path, "1")?;
@@ -157,22 +167,20 @@ pub fn pcipath_to_sysfs(root_bus_sysfs: &str, pcipath: &pci::Path) -> Result<Str
         let bridgebuspath = format!("{}{}/pci_bus", root_bus_sysfs, relpath);
         let mut files: Vec<_> = fs::read_dir(&bridgebuspath)?.collect();
 
-        match files.pop() {
-            Some(busfile) if files.is_empty() => {
-                bus = busfile?
-                    .file_name()
-                    .into_string()
-                    .map_err(|e| anyhow!("Bad filename under {}: {:?}", &bridgebuspath, e))?;
-            }
-            _ => {
-                return Err(anyhow!(
-                    "Expected exactly one PCI bus in {}, got {} instead",
-                    bridgebuspath,
-                    // Adjust to original value as we've already popped
-                    files.len() + 1
-                ));
-            }
-        };
+        if files.len() != 1 {
+            return Err(anyhow!(
+                "Expected exactly one PCI bus in {}, got {} instead",
+                bridgebuspath,
+                files.len()
+            ));
+        }
+
+        // unwrap is safe, because of the length test above
+        let busfile = files.pop().unwrap()?;
+        bus = busfile
+            .file_name()
+            .into_string()
+            .map_err(|e| anyhow!("Bad filename under {}: {:?}", &bridgebuspath, e))?;
     }
 
     Ok(relpath)
@@ -220,9 +228,8 @@ impl VirtioBlkPciMatcher {
     fn new(relpath: &str) -> VirtioBlkPciMatcher {
         let root_bus = create_pci_root_bus_path();
         let re = format!(r"^{}{}/virtio[0-9]+/block/", root_bus, relpath);
-
         VirtioBlkPciMatcher {
-            rex: Regex::new(&re).expect("BUG: failed to compile VirtioBlkPciMatcher regex"),
+            rex: Regex::new(&re).unwrap(),
         }
     }
 }
@@ -260,7 +267,7 @@ impl VirtioBlkCCWMatcher {
             root_bus_path, device
         );
         VirtioBlkCCWMatcher {
-            rex: Regex::new(&re).expect("BUG: failed to compile VirtioBlkCCWMatcher regex"),
+            rex: Regex::new(&re).unwrap(),
         }
     }
 }
@@ -416,15 +423,12 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
 
     for entry in fs::read_dir(SYSFS_SCSI_HOST_PATH)? {
         let host = entry?.file_name();
-
-        let host_str = host.to_str().ok_or_else(|| {
-            anyhow!(
-                "failed to convert directory entry to unicode for file {:?}",
-                host
-            )
-        })?;
-
-        let scan_path = PathBuf::from(&format!("{}/{}/{}", SYSFS_SCSI_HOST_PATH, host_str, "scan"));
+        let scan_path = format!(
+            "{}/{}/{}",
+            SYSFS_SCSI_HOST_PATH,
+            host.to_str().unwrap(),
+            "scan"
+        );
 
         fs::write(scan_path, &scan_data)?;
     }
@@ -432,201 +436,91 @@ fn scan_scsi_bus(scsi_addr: &str) -> Result<()> {
     Ok(())
 }
 
-#[derive(Debug, Clone)]
-struct DevNumUpdate {
-    // the major and minor numbers for the device within the guest
-    guest_major: i64,
-    guest_minor: i64,
-}
-
-impl DevNumUpdate {
-    fn from_vm_path<T: AsRef<Path>>(vm_path: T) -> Result<Self> {
-        let vm_path = vm_path.as_ref();
-
-        if !vm_path.exists() {
-            return Err(anyhow!("VM device path {:?} doesn't exist", vm_path));
-        }
-
-        let devid = fs::metadata(vm_path)?.rdev();
-        let guest_major = stat::major(devid) as i64;
-        let guest_minor = stat::minor(devid) as i64;
-
-        Ok(DevNumUpdate {
-            guest_major,
-            guest_minor,
-        })
-    }
-}
-
-// Represents the device-node and resource related updates to the OCI
-// spec needed for a particular device
-#[derive(Debug, Clone)]
-struct DevUpdate {
-    num: DevNumUpdate,
-    // an optional new path to update the device to in the "inner" container
-    // specification
-    final_path: Option<String>,
-}
-
-impl DevUpdate {
-    fn from_vm_path<T: AsRef<Path>>(vm_path: T, final_path: String) -> Result<Self> {
-        Ok(DevUpdate {
-            final_path: Some(final_path),
-            ..DevNumUpdate::from_vm_path(vm_path)?.into()
-        })
-    }
-}
-
-impl From<DevNumUpdate> for DevUpdate {
-    fn from(num: DevNumUpdate) -> Self {
-        DevUpdate {
-            num,
-            final_path: None,
-        }
-    }
-}
-
-// Represents the updates to the OCI spec needed for a particular device
-#[derive(Debug, Clone, Default)]
-struct SpecUpdate {
-    dev: Option<DevUpdate>,
-    // optional corrections for PCI addresses
-    pci: Vec<(pci::Address, pci::Address)>,
-}
-
-impl<T: Into<DevUpdate>> From<T> for SpecUpdate {
-    fn from(dev: T) -> Self {
-        SpecUpdate {
-            dev: Some(dev.into()),
-            pci: Vec::new(),
-        }
-    }
-}
-
-// update_spec_devices updates the device list in the OCI spec to make
+// update_spec_device updates the device list in the OCI spec to make
 // it include details appropriate for the VM, instead of the host.  It
-// is given a map of (container_path => update) where:
-//     container_path: the path to the device in the original OCI spec
-//     update: information on changes to make to the device
+// is given the host path to the device (to locate the device in the
+// original OCI spec) and the VM path which it uses to determine the
+// VM major/minor numbers, and the final path with which to present
+// the device in the (inner) container
 #[instrument]
-fn update_spec_devices(spec: &mut Spec, mut updates: HashMap<&str, DevUpdate>) -> Result<()> {
+fn update_spec_device(
+    spec: &mut Spec,
+    devidx: &DevIndex,
+    host_path: &str,
+    vm_path: &str,
+    final_path: &str,
+) -> Result<()> {
+    let major_id: c_uint;
+    let minor_id: c_uint;
+
+    // If no container_path is provided, we won't be able to match and
+    // update the device in the OCI spec device list. This is an error.
+    if host_path.is_empty() {
+        return Err(anyhow!("Host path cannot empty for device"));
+    }
+
     let linux = spec
         .linux
         .as_mut()
-        .ok_or_else(|| anyhow!("Spec didn't contain linux field"))?;
-    let mut res_updates = HashMap::<(&str, i64, i64), DevNumUpdate>::with_capacity(updates.len());
+        .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
-    for specdev in &mut linux.devices {
-        if let Some(update) = updates.remove(specdev.path.as_str()) {
-            let host_major = specdev.major;
-            let host_minor = specdev.minor;
+    if !Path::new(vm_path).exists() {
+        return Err(anyhow!("vm_path:{} doesn't exist", vm_path));
+    }
+
+    let meta = fs::metadata(vm_path)?;
+    let dev_id = meta.rdev();
+    unsafe {
+        major_id = major(dev_id);
+        minor_id = minor(dev_id);
+    }
+
+    info!(
+        sl!(),
+        "update_spec_device(): vm_path={}, major: {}, minor: {}\n", vm_path, major_id, minor_id
+    );
+
+    if let Some(idxdata) = devidx.0.get(host_path) {
+        let dev = &mut linux.devices[idxdata.idx];
+        let host_major = dev.major;
+        let host_minor = dev.minor;
+
+        dev.major = major_id as i64;
+        dev.minor = minor_id as i64;
+        dev.path = final_path.to_string();
+
+        info!(
+            sl!(),
+            "change the device from path: {} major: {} minor: {} to vm device path: {} major: {} minor: {}",
+            host_path,
+            host_major,
+            host_minor,
+            dev.path,
+            dev.major,
+            dev.minor,
+        );
+
+        // Resources must be updated since they are used to identify
+        // the device in the devices cgroup.
+        for ridx in &idxdata.residx {
+            // unwrap is safe, because residx would be empty if there
+            // were no resources
+            let res = &mut linux.resources.as_mut().unwrap().devices[*ridx];
+            res.major = Some(major_id as i64);
+            res.minor = Some(minor_id as i64);
 
             info!(
                 sl!(),
-                "update_spec_devices() updating device";
-                "container_path" => &specdev.path,
-                "type" => &specdev.r#type,
-                "host_major" => host_major,
-                "host_minor" => host_minor,
-                "guest_major" => update.num.guest_major,
-                "guest_minor" => update.num.guest_minor,
-                "final_path" => update.final_path.as_ref(),
+                "set resources for device major: {} minor: {}\n", major_id, minor_id
             );
-
-            specdev.major = update.num.guest_major;
-            specdev.minor = update.num.guest_minor;
-            if let Some(final_path) = update.final_path {
-                specdev.path = final_path;
-            }
-
-            if res_updates
-                .insert(
-                    (specdev.r#type.as_str(), host_major, host_minor),
-                    update.num,
-                )
-                .is_some()
-            {
-                return Err(anyhow!(
-                    "Conflicting resource updates for host_major={} host_minor={}",
-                    host_major,
-                    host_minor
-                ));
-            }
         }
+        Ok(())
+    } else {
+        Err(anyhow!(
+            "Should have found a matching device {} in the spec",
+            vm_path
+        ))
     }
-
-    // Make sure we applied all of our updates
-    if !updates.is_empty() {
-        return Err(anyhow!(
-            "Missing devices in OCI spec: {:?}",
-            updates
-                .keys()
-                .map(|d| format!("{:?}", d))
-                .collect::<Vec<_>>()
-                .join(" ")
-        ));
-    }
-
-    if let Some(resources) = linux.resources.as_mut() {
-        for r in &mut resources.devices {
-            if let (Some(host_major), Some(host_minor)) = (r.major, r.minor) {
-                if let Some(update) = res_updates.get(&(r.r#type.as_str(), host_major, host_minor))
-                {
-                    info!(
-                        sl!(),
-                        "update_spec_devices() updating resource";
-                        "type" => &r.r#type,
-                        "host_major" => host_major,
-                        "host_minor" => host_minor,
-                        "guest_major" => update.guest_major,
-                        "guest_minor" => update.guest_minor,
-                    );
-
-                    r.major = Some(update.guest_major);
-                    r.minor = Some(update.guest_minor);
-                }
-            }
-        }
-    }
-
-    Ok(())
-}
-
-// update_spec_pci PCI addresses in the OCI spec to be guest addresses
-// instead of host addresses.  It is given a map of (host address =>
-// guest address)
-#[instrument]
-fn update_spec_pci(spec: &mut Spec, updates: HashMap<pci::Address, pci::Address>) -> Result<()> {
-    // Correct PCI addresses in the environment
-    if let Some(process) = spec.process.as_mut() {
-        for envvar in process.env.iter_mut() {
-            let eqpos = envvar
-                .find('=')
-                .ok_or_else(|| anyhow!("Malformed OCI env entry {:?}", envvar))?;
-
-            let (name, eqval) = envvar.split_at(eqpos);
-            let val = &eqval[1..];
-
-            if !name.starts_with("PCIDEVICE_") {
-                continue;
-            }
-
-            let mut guest_addrs = Vec::<String>::new();
-
-            for host_addr in val.split(',') {
-                let host_addr = pci::Address::from_str(host_addr)
-                    .with_context(|| format!("Can't parse {} environment variable", name))?;
-                let guest_addr = updates
-                    .get(&host_addr)
-                    .ok_or_else(|| anyhow!("Unable to translate host PCI address {}", host_addr))?;
-                guest_addrs.push(format!("{}", guest_addr));
-            }
-
-            envvar.replace_range(eqpos + 1.., guest_addrs.join(",").as_str());
-        }
-    }
-
-    Ok(())
 }
 
 // device.Id should be the predicted device name (vda, vdb, ...)
@@ -634,25 +528,43 @@ fn update_spec_pci(spec: &mut Spec, updates: HashMap<pci::Address, pci::Address>
 #[instrument]
 async fn virtiommio_blk_device_handler(
     device: &Device,
+    spec: &mut Spec,
     _sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
+    devidx: &DevIndex,
+) -> Result<()> {
     if device.vm_path.is_empty() {
         return Err(anyhow!("Invalid path for virtio mmio blk device"));
     }
 
-    Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 // device.Id should be a PCI path string
 #[instrument]
 async fn virtio_blk_device_handler(
     device: &Device,
+    spec: &mut Spec,
     sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
+    devidx: &DevIndex,
+) -> Result<()> {
+    let mut dev = device.clone();
     let pcipath = pci::Path::from_str(&device.id)?;
-    let vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
 
-    Ok(DevNumUpdate::from_vm_path(vm_path)?.into())
+    dev.vm_path = get_virtio_blk_pci_device_name(sandbox, &pcipath).await?;
+
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 // device.id should be a CCW path string
@@ -660,17 +572,30 @@ async fn virtio_blk_device_handler(
 #[instrument]
 async fn virtio_blk_ccw_device_handler(
     device: &Device,
+    spec: &mut Spec,
     sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
+    devidx: &DevIndex,
+) -> Result<()> {
+    let mut dev = device.clone();
     let ccw_device = ccw::Device::from_str(&device.id)?;
-    let vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
-
-    Ok(DevNumUpdate::from_vm_path(vm_path)?.into())
+    dev.vm_path = get_virtio_blk_ccw_device_name(sandbox, &ccw_device).await?;
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[cfg(not(target_arch = "s390x"))]
 #[instrument]
-async fn virtio_blk_ccw_device_handler(_: &Device, _: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+async fn virtio_blk_ccw_device_handler(
+    _: &Device,
+    _: &mut Spec,
+    _: &Arc<Mutex<Sandbox>>,
+    _: &DevIndex,
+) -> Result<()> {
     Err(anyhow!("CCW is only supported on s390x"))
 }
 
@@ -678,23 +603,39 @@ async fn virtio_blk_ccw_device_handler(_: &Device, _: &Arc<Mutex<Sandbox>>) -> R
 #[instrument]
 async fn virtio_scsi_device_handler(
     device: &Device,
+    spec: &mut Spec,
     sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
-    let vm_path = get_scsi_device_name(sandbox, &device.id).await?;
-
-    Ok(DevNumUpdate::from_vm_path(vm_path)?.into())
+    devidx: &DevIndex,
+) -> Result<()> {
+    let mut dev = device.clone();
+    dev.vm_path = get_scsi_device_name(sandbox, &device.id).await?;
+    update_spec_device(
+        spec,
+        devidx,
+        &dev.container_path,
+        &dev.vm_path,
+        &dev.container_path,
+    )
 }
 
 #[instrument]
 async fn virtio_nvdimm_device_handler(
     device: &Device,
+    spec: &mut Spec,
     _sandbox: &Arc<Mutex<Sandbox>>,
-) -> Result<SpecUpdate> {
+    devidx: &DevIndex,
+) -> Result<()> {
     if device.vm_path.is_empty() {
         return Err(anyhow!("Invalid path for nvdimm device"));
     }
 
-    Ok(DevNumUpdate::from_vm_path(&device.vm_path)?.into())
+    update_spec_device(
+        spec,
+        devidx,
+        &device.container_path,
+        &device.vm_path,
+        &device.container_path,
+    )
 }
 
 fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
@@ -712,53 +653,80 @@ fn split_vfio_option(opt: &str) -> Option<(&str, &str)> {
 // Each option should have the form "DDDD:BB:DD.F=<pcipath>"
 //     DDDD:BB:DD.F is the device's PCI address in the host
 //     <pcipath> is a PCI path to the device in the guest (see pci.rs)
-async fn vfio_device_handler(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+async fn vfio_device_handler(
+    device: &Device,
+    spec: &mut Spec,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devidx: &DevIndex,
+) -> Result<()> {
     let vfio_in_guest = device.field_type != DRIVER_VFIO_GK_TYPE;
-    let mut pci_fixups = Vec::<(pci::Address, pci::Address)>::new();
     let mut group = None;
 
     for opt in device.options.iter() {
-        let (host, pcipath) =
+        let (_, pcipath) =
             split_vfio_option(opt).ok_or_else(|| anyhow!("Malformed VFIO option {:?}", opt))?;
-        let host =
-            pci::Address::from_str(host).context("Bad host PCI address in VFIO option {:?}")?;
         let pcipath = pci::Path::from_str(pcipath)?;
 
         let guestdev = wait_for_pci_device(sandbox, &pcipath).await?;
         if vfio_in_guest {
             pci_driver_override(SYSFS_BUS_PCI_PATH, guestdev, "vfio-pci")?;
 
-            // Devices must have an IOMMU group to be usable via VFIO
-            let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?
-                .ok_or_else(|| anyhow!("{} has no IOMMU group", guestdev))?;
-
-            if let Some(g) = group {
-                if g != devgroup {
-                    return Err(anyhow!("{} is not in guest IOMMU group {}", guestdev, g));
-                }
+            let devgroup = pci_iommu_group(SYSFS_BUS_PCI_PATH, guestdev)?;
+            if devgroup.is_none() {
+                // Devices must have an IOMMU group to be usable via VFIO
+                return Err(anyhow!("{} has no IOMMU group", guestdev));
             }
 
-            group = Some(devgroup);
+            if group.is_some() && group != devgroup {
+                // If PCI devices associated with the same VFIO device
+                // (and therefore group) in the host don't end up in
+                // the same group in the guest, something has gone
+                // horribly wrong
+                return Err(anyhow!(
+                    "{} is not in guest IOMMU group {}",
+                    guestdev,
+                    group.unwrap()
+                ));
+            }
 
-            pci_fixups.push((host, guestdev));
+            group = devgroup;
         }
     }
 
-    let dev_update = if vfio_in_guest {
+    if vfio_in_guest {
         // If there are any devices at all, logic above ensures that group is not None
-        let group = group.ok_or_else(|| anyhow!("failed to get VFIO group: {:?}"))?;
+        let group = group.unwrap();
+        let vmpath = get_vfio_device_name(sandbox, group).await?;
 
-        let vm_path = get_vfio_device_name(sandbox, group).await?;
+        update_spec_device(spec, devidx, &device.container_path, &vmpath, &vmpath)?;
+    }
 
-        Some(DevUpdate::from_vm_path(&vm_path, vm_path.clone())?)
-    } else {
-        None
-    };
+    Ok(())
+}
 
-    Ok(SpecUpdate {
-        dev: dev_update,
-        pci: pci_fixups,
-    })
+impl DevIndex {
+    fn new(spec: &Spec) -> DevIndex {
+        let mut map = HashMap::new();
+
+        if let Some(linux) = spec.linux.as_ref() {
+            for (i, d) in linux.devices.iter().enumerate() {
+                let mut residx = Vec::new();
+
+                if let Some(linuxres) = linux.resources.as_ref() {
+                    for (j, r) in linuxres.devices.iter().enumerate() {
+                        if r.r#type == d.r#type
+                            && r.major == Some(d.major)
+                            && r.minor == Some(d.minor)
+                        {
+                            residx.push(j);
+                        }
+                    }
+                }
+                map.insert(d.path.clone(), DevIndexEntry { idx: i, residx });
+            }
+        }
+        DevIndex(map)
+    }
 }
 
 #[instrument]
@@ -767,40 +735,22 @@ pub async fn add_devices(
     spec: &mut Spec,
     sandbox: &Arc<Mutex<Sandbox>>,
 ) -> Result<()> {
-    let mut dev_updates = HashMap::<&str, DevUpdate>::with_capacity(devices.len());
-    let mut pci_updates = HashMap::<pci::Address, pci::Address>::new();
+    let devidx = DevIndex::new(spec);
 
     for device in devices.iter() {
-        let update = add_device(device, sandbox).await?;
-        if let Some(dev_update) = update.dev {
-            if dev_updates
-                .insert(&device.container_path, dev_update)
-                .is_some()
-            {
-                return Err(anyhow!(
-                    "Conflicting device updates for {}",
-                    &device.container_path
-                ));
-            }
-
-            for (host, guest) in update.pci {
-                if let Some(other_guest) = pci_updates.insert(host, guest) {
-                    return Err(anyhow!(
-                        "Conflicting guest address for host device {} ({} versus {})",
-                        host,
-                        guest,
-                        other_guest
-                    ));
-                }
-            }
-        }
+        add_device(device, spec, sandbox, &devidx).await?;
     }
 
-    update_spec_devices(spec, dev_updates)
+    Ok(())
 }
 
 #[instrument]
-async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<SpecUpdate> {
+async fn add_device(
+    device: &Device,
+    spec: &mut Spec,
+    sandbox: &Arc<Mutex<Sandbox>>,
+    devidx: &DevIndex,
+) -> Result<()> {
     // log before validation to help with debugging gRPC protocol version differences.
     info!(sl!(), "device-id: {}, device-type: {}, device-vm-path: {}, device-container-path: {}, device-options: {:?}",
           device.id, device.field_type, device.vm_path, device.container_path, device.options);
@@ -818,12 +768,14 @@ async fn add_device(device: &Device, sandbox: &Arc<Mutex<Sandbox>>) -> Result<Sp
     }
 
     match device.field_type.as_str() {
-        DRIVER_BLK_TYPE => virtio_blk_device_handler(device, sandbox).await,
-        DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, sandbox).await,
-        DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, sandbox).await,
-        DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, sandbox).await,
-        DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, sandbox).await,
-        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => vfio_device_handler(device, sandbox).await,
+        DRIVER_BLK_TYPE => virtio_blk_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_BLK_CCW_TYPE => virtio_blk_ccw_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_MMIO_BLK_TYPE => virtiommio_blk_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_NVDIMM_TYPE => virtio_nvdimm_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_SCSI_TYPE => virtio_scsi_device_handler(device, spec, sandbox, devidx).await,
+        DRIVER_VFIO_GK_TYPE | DRIVER_VFIO_TYPE => {
+            vfio_device_handler(device, spec, sandbox, devidx).await
+        }
         _ => Err(anyhow!("Unknown device type {}", device.field_type)),
     }
 }
@@ -843,8 +795,11 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
         .as_mut()
         .ok_or_else(|| anyhow!("Spec didn't container linux field"))?;
 
-    let resources = linux.resources.get_or_insert(LinuxResources::default());
+    if linux.resources.is_none() {
+        linux.resources = Some(LinuxResources::default());
+    }
 
+    let resources = linux.resources.as_mut().unwrap();
     resources.devices.push(LinuxDeviceCgroup {
         allow: false,
         major: Some(major),
@@ -860,8 +815,7 @@ pub fn update_device_cgroup(spec: &mut Spec) -> Result<()> {
 mod tests {
     use super::*;
     use crate::uevent::spawn_test_watcher;
-    use oci::{Linux, Process};
-    use std::iter::FromIterator;
+    use oci::Linux;
     use tempfile::tempdir;
 
     #[test]
@@ -886,36 +840,28 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_devices() {
+    fn test_update_spec_device() {
         let (major, minor) = (7, 2);
         let mut spec = Spec::default();
 
-        // vm_path empty
-        let update = DevNumUpdate::from_vm_path("");
-        assert!(update.is_err());
+        // container_path empty
+        let container_path = "";
+        let vm_path = "";
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
+        assert!(res.is_err());
 
         // linux is empty
         let container_path = "/dev/null";
-        let vm_path = "/dev/null";
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevNumUpdate::from_vm_path(vm_path).unwrap().into(),
-            )]),
-        );
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux = Some(Linux::default());
 
-        // linux.devices doesn't contain the updated device
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevNumUpdate::from_vm_path(vm_path).unwrap().into(),
-            )]),
-        );
+        // linux.devices is empty
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_err());
 
         spec.linux.as_mut().unwrap().devices = vec![oci::LinuxDevice {
@@ -925,14 +871,16 @@ mod tests {
             ..oci::LinuxDevice::default()
         }];
 
+        // vm_path empty
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
+        assert!(res.is_err());
+
+        let vm_path = "/dev/null";
+
         // guest and host path are not the same
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevNumUpdate::from_vm_path(vm_path).unwrap().into(),
-            )]),
-        );
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(
             res.is_err(),
             "container_path={:?} vm_path={:?} spec={:?}",
@@ -944,13 +892,8 @@ mod tests {
         spec.linux.as_mut().unwrap().devices[0].path = container_path.to_string();
 
         // spec.linux.resources is empty
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevNumUpdate::from_vm_path(vm_path).unwrap().into(),
-            )]),
-        );
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // update both devices and cgroup lists
@@ -970,18 +913,13 @@ mod tests {
             ..oci::LinuxResources::default()
         });
 
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevNumUpdate::from_vm_path(vm_path).unwrap().into(),
-            )]),
-        );
+        let devidx = DevIndex::new(&spec);
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
     }
 
     #[test]
-    fn test_update_spec_devices_guest_host_conflict() {
+    fn test_update_spec_device_guest_host_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
         let zero_rdev = fs::metadata("/dev/zero").unwrap().rdev();
         let full_rdev = fs::metadata("/dev/full").unwrap().rdev();
@@ -1030,6 +968,7 @@ mod tests {
             }),
             ..Spec::default()
         };
+        let devidx = DevIndex::new(&spec);
 
         let container_path_a = "/dev/a";
         let vm_path_a = "/dev/zero";
@@ -1055,17 +994,34 @@ mod tests {
         assert_eq!(Some(host_major_b), specresources.devices[1].major);
         assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
 
-        let updates = HashMap::from_iter(vec![
-            (
-                container_path_a,
-                DevNumUpdate::from_vm_path(vm_path_a).unwrap().into(),
-            ),
-            (
-                container_path_b,
-                DevNumUpdate::from_vm_path(vm_path_b).unwrap().into(),
-            ),
-        ]);
-        let res = update_spec_devices(&mut spec, updates);
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_a,
+            vm_path_a,
+            container_path_a,
+        );
+        assert!(res.is_ok());
+
+        let specdevices = &spec.linux.as_ref().unwrap().devices;
+        assert_eq!(guest_major_a, specdevices[0].major);
+        assert_eq!(guest_minor_a, specdevices[0].minor);
+        assert_eq!(host_major_b, specdevices[1].major);
+        assert_eq!(host_minor_b, specdevices[1].minor);
+
+        let specresources = spec.linux.as_ref().unwrap().resources.as_ref().unwrap();
+        assert_eq!(Some(guest_major_a), specresources.devices[0].major);
+        assert_eq!(Some(guest_minor_a), specresources.devices[0].minor);
+        assert_eq!(Some(host_major_b), specresources.devices[1].major);
+        assert_eq!(Some(host_minor_b), specresources.devices[1].minor);
+
+        let res = update_spec_device(
+            &mut spec,
+            &devidx,
+            container_path_b,
+            vm_path_b,
+            container_path_b,
+        );
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -1082,7 +1038,7 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_devices_char_block_conflict() {
+    fn test_update_spec_device_char_block_conflict() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
 
         let guest_major = stat::major(null_rdev) as i64;
@@ -1129,6 +1085,7 @@ mod tests {
             }),
             ..Spec::default()
         };
+        let devidx = DevIndex::new(&spec);
 
         let container_path = "/dev/char";
         let vm_path = "/dev/null";
@@ -1139,13 +1096,7 @@ mod tests {
         assert_eq!(Some(host_major), specresources.devices[1].major);
         assert_eq!(Some(host_minor), specresources.devices[1].minor);
 
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevNumUpdate::from_vm_path(vm_path).unwrap().into(),
-            )]),
-        );
+        let res = update_spec_device(&mut spec, &devidx, container_path, vm_path, container_path);
         assert!(res.is_ok());
 
         // Only the char device, not the block device should be updated
@@ -1157,19 +1108,19 @@ mod tests {
     }
 
     #[test]
-    fn test_update_spec_devices_final_path() {
+    fn test_update_spec_device_final_path() {
         let null_rdev = fs::metadata("/dev/null").unwrap().rdev();
         let guest_major = stat::major(null_rdev) as i64;
         let guest_minor = stat::minor(null_rdev) as i64;
 
-        let container_path = "/dev/original";
+        let host_path = "/dev/host";
         let host_major: i64 = 99;
         let host_minor: i64 = 99;
 
         let mut spec = Spec {
             linux: Some(Linux {
                 devices: vec![oci::LinuxDevice {
-                    path: container_path.to_string(),
+                    path: host_path.to_string(),
                     r#type: "c".to_string(),
                     major: host_major,
                     minor: host_minor,
@@ -1179,17 +1130,12 @@ mod tests {
             }),
             ..Spec::default()
         };
+        let devidx = DevIndex::new(&spec);
 
         let vm_path = "/dev/null";
-        let final_path = "/dev/new";
+        let final_path = "/dev/final";
 
-        let res = update_spec_devices(
-            &mut spec,
-            HashMap::from_iter(vec![(
-                container_path,
-                DevUpdate::from_vm_path(vm_path, final_path.to_string()).unwrap(),
-            )]),
-        );
+        let res = update_spec_device(&mut spec, &devidx, host_path, vm_path, final_path);
         assert!(res.is_ok());
 
         let specdevices = &spec.linux.as_ref().unwrap().devices;
@@ -1198,48 +1144,6 @@ mod tests {
         assert_eq!(final_path, specdevices[0].path);
     }
 
-    #[test]
-    fn test_update_spec_pci() {
-        let example_map = [
-            // Each is a host,guest pair of pci addresses
-            ("0000:1a:01.0", "0000:01:01.0"),
-            ("0000:1b:02.0", "0000:01:02.0"),
-            // This one has the same host address as guest address
-            // above, to test that we're not double-translating
-            ("0000:01:01.0", "ffff:02:1f.7"),
-        ];
-
-        let mut spec = Spec {
-            process: Some(Process {
-                env: vec![
-                    "PCIDEVICE_x=0000:1a:01.0,0000:1b:02.0".to_string(),
-                    "PCIDEVICE_y=0000:01:01.0".to_string(),
-                    "NOTAPCIDEVICE_blah=abcd:ef:01.0".to_string(),
-                ],
-                ..Process::default()
-            }),
-            ..Spec::default()
-        };
-
-        let pci_fixups = example_map
-            .iter()
-            .map(|(h, g)| {
-                (
-                    pci::Address::from_str(h).unwrap(),
-                    pci::Address::from_str(g).unwrap(),
-                )
-            })
-            .collect();
-
-        let res = update_spec_pci(&mut spec, pci_fixups);
-        assert!(res.is_ok());
-
-        let env = &spec.process.as_ref().unwrap().env;
-        assert_eq!(env[0], "PCIDEVICE_x=0000:01:01.0,0000:01:02.0");
-        assert_eq!(env[1], "PCIDEVICE_y=ffff:02:1f.7");
-        assert_eq!(env[2], "NOTAPCIDEVICE_blah=abcd:ef:01.0");
-    }
-
     #[test]
     fn test_pcipath_to_sysfs() {
         let testdir = tempdir().expect("failed to create tmpdir");

src/agent/src/main.rs

@@ -113,10 +113,10 @@ async fn create_logger_task(rfd: RawFd, vsock_port: u32, shutdown: Receiver<bool
         )?;
 
         let addr = SockAddr::new_vsock(libc::VMADDR_CID_ANY, vsock_port);
-        socket::bind(listenfd, &addr)?;
-        socket::listen(listenfd, 1)?;
+        socket::bind(listenfd, &addr).unwrap();
+        socket::listen(listenfd, 1).unwrap();
 
-        writer = Box::new(util::get_vsock_stream(listenfd).await?);
+        writer = Box::new(util::get_vsock_stream(listenfd).await.unwrap());
     } else {
         writer = Box::new(tokio::io::stdout());
     }
@@ -326,7 +326,7 @@ async fn start_sandbox(
     sandbox.lock().await.sender = Some(tx);
 
     // vsock:///dev/vsock, port
-    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str())?;
+    let mut server = rpc::start(sandbox.clone(), config.server_addr.as_str());
     server.start().await?;
 
     rx.await?;

src/agent/src/metrics.rs

@@ -8,7 +8,6 @@ extern crate procfs;
 use prometheus::{Encoder, Gauge, GaugeVec, IntCounter, TextEncoder};
 
 use anyhow::Result;
-use slog::warn;
 use tracing::instrument;
 
 const NAMESPACE_KATA_AGENT: &str = "kata_agent";
@@ -24,50 +23,50 @@ macro_rules! sl {
 lazy_static! {
 
     static ref     AGENT_SCRAPE_COUNT: IntCounter =
-    prometheus::register_int_counter!(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count").as_ref(), "Metrics scrape count").unwrap();
+    prometheus::register_int_counter!(format!("{}_{}",NAMESPACE_KATA_AGENT,"scrape_count"), "Metrics scrape count").unwrap();
 
     static ref     AGENT_THREADS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads").as_ref(), "Agent process threads").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"threads"), "Agent process threads").unwrap();
 
     static ref     AGENT_TOTAL_TIME: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time").as_ref(), "Agent process total time").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_time"), "Agent process total time").unwrap();
 
     static ref     AGENT_TOTAL_VM: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm").as_ref(), "Agent process total VM size").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_vm"), "Agent process total VM size").unwrap();
 
     static ref     AGENT_TOTAL_RSS: Gauge =
-    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss").as_ref(), "Agent process total RSS size").unwrap();
+    prometheus::register_gauge!(format!("{}_{}",NAMESPACE_KATA_AGENT,"total_rss"), "Agent process total RSS size").unwrap();
 
     static ref     AGENT_PROC_STATUS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status").as_ref(), "Agent process status.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_status"), "Agent process status.", &["item"]).unwrap();
 
     static ref     AGENT_IO_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat").as_ref(), "Agent process IO statistics.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"io_stat"), "Agent process IO statistics.", &["item"]).unwrap();
 
     static ref     AGENT_PROC_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat").as_ref(), "Agent process statistics.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_AGENT,"proc_stat"), "Agent process statistics.", &["item"]).unwrap();
 
     // guest os metrics
     static ref     GUEST_LOAD: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"load").as_ref() , "Guest system load.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"load") , "Guest system load.", &["item"]).unwrap();
 
     static ref     GUEST_TASKS: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks").as_ref() , "Guest system load.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"tasks") , "Guest system load.", &["item"]).unwrap();
 
     static ref     GUEST_CPU_TIME: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time").as_ref() , "Guest CPU statistics.", &["cpu","item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"cpu_time") , "Guest CPU statistics.", &["cpu","item"]).unwrap();
 
     static ref     GUEST_VM_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat").as_ref() , "Guest virtual memory statistics.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"vm_stat") , "Guest virtual memory statistics.", &["item"]).unwrap();
 
     static ref     GUEST_NETDEV_STAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat").as_ref() , "Guest net devices statistics.", &["interface","item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"netdev_stat") , "Guest net devices statistics.", &["interface","item"]).unwrap();
 
     static ref     GUEST_DISKSTAT: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat").as_ref() , "Disks statistics in system.", &["disk","item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"diskstat") , "Disks statistics in system.", &["disk","item"]).unwrap();
 
     static ref     GUEST_MEMINFO: GaugeVec =
-    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo").as_ref() , "Statistics about memory usage in the system.", &["item"]).unwrap();
+    prometheus::register_gauge_vec!(format!("{}_{}",NAMESPACE_KATA_GUEST,"meminfo") , "Statistics about memory usage in the system.", &["item"]).unwrap();
 }
 
 #[instrument]
@@ -75,7 +74,7 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
     AGENT_SCRAPE_COUNT.inc();
 
     // update agent process metrics
-    update_agent_metrics()?;
+    update_agent_metrics();
 
     // update guest os metrics
     update_guest_metrics();
@@ -85,26 +84,23 @@ pub fn get_metrics(_: &protocols::agent::GetMetricsRequest) -> Result<String> {
 
     let mut buffer = Vec::new();
     let encoder = TextEncoder::new();
-    encoder.encode(&metric_families, &mut buffer)?;
+    encoder.encode(&metric_families, &mut buffer).unwrap();
 
-    Ok(String::from_utf8(buffer)?)
+    Ok(String::from_utf8(buffer).unwrap())
 }
 
 #[instrument]
-fn update_agent_metrics() -> Result<()> {
+fn update_agent_metrics() {
     let me = procfs::process::Process::myself();
 
-    let me = match me {
-        Ok(p) => p,
-        Err(e) => {
-            // FIXME: return Ok for all errors?
-            warn!(sl!(), "failed to create process instance: {:?}", e);
+    if let Err(err) = me {
+        error!(sl!(), "failed to create process instance: {:?}", err);
+        return;
+    }
 
-            return Ok(());
-        }
-    };
+    let me = me.unwrap();
 
-    let tps = procfs::ticks_per_second()?;
+    let tps = procfs::ticks_per_second().unwrap();
 
     // process total time
     AGENT_TOTAL_TIME.set((me.stat.utime + me.stat.stime) as f64 / (tps as f64));
@@ -113,7 +109,7 @@ fn update_agent_metrics() -> Result<()> {
     AGENT_TOTAL_VM.set(me.stat.vsize as f64);
 
     // Total resident set
-    let page_size = procfs::page_size()? as f64;
+    let page_size = procfs::page_size().unwrap() as f64;
     AGENT_TOTAL_RSS.set(me.stat.rss as f64 * page_size);
 
     // io
@@ -136,11 +132,11 @@ fn update_agent_metrics() -> Result<()> {
     }
 
     match me.status() {
-        Err(err) => error!(sl!(), "failed to get process status: {:?}", err),
+        Err(err) => {
+            info!(sl!(), "failed to get process status: {:?}", err);
+        }
         Ok(status) => set_gauge_vec_proc_status(&AGENT_PROC_STATUS, &status),
     }
-
-    Ok(())
 }
 
 #[instrument]
@@ -352,17 +348,17 @@ fn set_gauge_vec_cpu_time(gv: &prometheus::GaugeVec, cpu: &str, cpu_time: &procf
     gv.with_label_values(&[cpu, "idle"])
         .set(cpu_time.idle as f64);
     gv.with_label_values(&[cpu, "iowait"])
-        .set(cpu_time.iowait.unwrap_or(0.0) as f64);
+        .set(cpu_time.iowait.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "irq"])
-        .set(cpu_time.irq.unwrap_or(0.0) as f64);
+        .set(cpu_time.irq.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "softirq"])
-        .set(cpu_time.softirq.unwrap_or(0.0) as f64);
+        .set(cpu_time.softirq.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "steal"])
-        .set(cpu_time.steal.unwrap_or(0.0) as f64);
+        .set(cpu_time.steal.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest"])
-        .set(cpu_time.guest.unwrap_or(0.0) as f64);
+        .set(cpu_time.guest.unwrap_or(0) as f64);
     gv.with_label_values(&[cpu, "guest_nice"])
-        .set(cpu_time.guest_nice.unwrap_or(0.0) as f64);
+        .set(cpu_time.guest_nice.unwrap_or(0) as f64);
 }
 
 #[instrument]
@@ -474,7 +470,7 @@ fn set_gauge_vec_proc_status(gv: &prometheus::GaugeVec, status: &procfs::process
     gv.with_label_values(&["vmswap"])
         .set(status.vmswap.unwrap_or(0) as f64);
     gv.with_label_values(&["hugetlbpages"])
-        .set(status.hugetblpages.unwrap_or(0) as f64);
+        .set(status.hugetlbpages.unwrap_or(0) as f64);
     gv.with_label_values(&["voluntary_ctxt_switches"])
         .set(status.voluntary_ctxt_switches.unwrap_or(0) as f64);
     gv.with_label_values(&["nonvoluntary_ctxt_switches"])

src/agent/src/mount.rs

@@ -139,8 +139,8 @@ pub const STORAGE_HANDLER_LIST: &[&str] = &[
 
 #[instrument]
 pub fn baremount(
-    source: &Path,
-    destination: &Path,
+    source: &str,
+    destination: &str,
     fs_type: &str,
     flags: MsFlags,
     options: &str,
@@ -148,11 +148,11 @@ pub fn baremount(
 ) -> Result<()> {
     let logger = logger.new(o!("subsystem" => "baremount"));
 
-    if source.as_os_str().is_empty() {
+    if source.is_empty() {
         return Err(anyhow!("need mount source"));
     }
 
-    if destination.as_os_str().is_empty() {
+    if destination.is_empty() {
         return Err(anyhow!("need mount destination"));
     }
 
@@ -405,14 +405,18 @@ async fn bind_watcher_storage_handler(
     logger: &Logger,
     storage: &Storage,
     sandbox: Arc<Mutex<Sandbox>>,
+    cid: Option<String>,
 ) -> Result<()> {
     let mut locked = sandbox.lock().await;
-    let container_id = locked.id.clone();
 
-    locked
-        .bind_watcher
-        .add_container(container_id, iter::once(storage.clone()), logger)
-        .await
+    if let Some(cid) = cid {
+        locked
+            .bind_watcher
+            .add_container(cid, iter::once(storage.clone()), logger)
+            .await
+    } else {
+        Ok(())
+    }
 }
 
 // mount_storage performs the mount described by the storage structure.
@@ -444,19 +448,16 @@ fn mount_storage(logger: &Logger, storage: &Storage) -> Result<()> {
     let options_vec = options_vec.iter().map(String::as_str).collect();
     let (flags, options) = parse_mount_flags_and_options(options_vec);
 
-    let source = Path::new(&storage.source);
-    let mount_point = Path::new(&storage.mount_point);
-
     info!(logger, "mounting storage";
-    "mount-source" => source.display(),
-    "mount-destination" => mount_point.display(),
+    "mount-source:" => storage.source.as_str(),
+    "mount-destination" => storage.mount_point.as_str(),
     "mount-fstype"  => storage.fstype.as_str(),
     "mount-options" => options.as_str(),
     );
 
     baremount(
-        source,
-        mount_point,
+        storage.source.as_str(),
+        storage.mount_point.as_str(),
         storage.fstype.as_str(),
         flags,
         options.as_str(),
@@ -521,6 +522,7 @@ pub async fn add_storages(
     logger: Logger,
     storages: Vec<Storage>,
     sandbox: Arc<Mutex<Sandbox>>,
+    cid: Option<String>,
 ) -> Result<Vec<String>> {
     let mut mount_list = Vec::new();
 
@@ -551,7 +553,8 @@ pub async fn add_storages(
             }
             DRIVER_NVDIMM_TYPE => nvdimm_storage_handler(&logger, &storage, sandbox.clone()).await,
             DRIVER_WATCHABLE_BIND_TYPE => {
-                bind_watcher_storage_handler(&logger, &storage, sandbox.clone()).await?;
+                bind_watcher_storage_handler(&logger, &storage, sandbox.clone(), cid.clone())
+                    .await?;
                 // Don't register watch mounts, they're handled separately by the watcher.
                 Ok(String::new())
             }
@@ -582,10 +585,7 @@ fn mount_to_rootfs(logger: &Logger, m: &InitMount) -> Result<()> {
 
     fs::create_dir_all(Path::new(m.dest)).context("could not create directory")?;
 
-    let source = Path::new(m.src);
-    let dest = Path::new(m.dest);
-
-    baremount(source, dest, m.fstype, flags, &options, logger).or_else(|e| {
+    baremount(m.src, m.dest, m.fstype, flags, &options, logger).or_else(|e| {
         if m.src != "dev" {
             return Err(e);
         }
@@ -628,7 +628,8 @@ pub fn get_mount_fs_type_from_file(mount_file: &str, mount_point: &str) -> Resul
     let file = File::open(mount_file)?;
     let reader = BufReader::new(file);
 
-    let re = Regex::new(format!("device .+ mounted on {} with fstype (.+)", mount_point).as_str())?;
+    let re = Regex::new(format!("device .+ mounted on {} with fstype (.+)", mount_point).as_str())
+        .unwrap();
 
     // Read the file line by line using the lines() iterator from std::io::BufRead.
     for (_index, line) in reader.lines().enumerate() {
@@ -706,21 +707,20 @@ pub fn get_cgroup_mounts(
             }
         }
 
-        let subsystem_name = fields[0];
-
-        if subsystem_name.is_empty() {
+        if fields[0].is_empty() {
             continue;
         }
 
-        if subsystem_name == "devices" {
+        if fields[0] == "devices" {
             has_device_cgroup = true;
         }
 
-        if let Some((key, value)) = CGROUPS.get_key_value(subsystem_name) {
+        if let Some(value) = CGROUPS.get(&fields[0]) {
+            let key = CGROUPS.keys().find(|&&f| f == fields[0]).unwrap();
             cg_mounts.push(InitMount {
                 fstype: "cgroup",
                 src: "cgroup",
-                dest: value,
+                dest: *value,
                 options: vec!["nosuid", "nodev", "noexec", "relatime", key],
             });
         }
@@ -773,9 +773,10 @@ fn ensure_destination_file_exists(path: &Path) -> Result<()> {
         return Err(anyhow!("{:?} exists but is not a regular file", path));
     }
 
-    let dir = path
-        .parent()
-        .ok_or_else(|| anyhow!("failed to find parent path for {:?}", path))?;
+    // The only way parent() can return None is if the path is /,
+    // which always exists, so the test above will already have caught
+    // it, thus the unwrap() is safe
+    let dir = path.parent().unwrap();
 
     fs::create_dir_all(dir).context(format!("create_dir_all {:?}", dir))?;
 
@@ -942,10 +943,14 @@ mod tests {
                 std::fs::create_dir_all(d).expect("failed to created directory");
             }
 
-            let src = Path::new(&src_filename);
-            let dest = Path::new(&dest_filename);
-
-            let result = baremount(src, dest, d.fs_type, d.flags, d.options, &logger);
+            let result = baremount(
+                &src_filename,
+                &dest_filename,
+                d.fs_type,
+                d.flags,
+                d.options,
+                &logger,
+            );
 
             let msg = format!("{}: result: {:?}", msg, result);
 
@@ -1022,11 +1027,15 @@ mod tests {
                 .unwrap_or_else(|_| panic!("failed to create directory {}", d));
         }
 
-        let src = Path::new(mnt_src_filename);
-        let dest = Path::new(mnt_dest_filename);
-
         // Create an actual mount
-        let result = baremount(src, dest, "bind", MsFlags::MS_BIND, "", &logger);
+        let result = baremount(
+            mnt_src_filename,
+            mnt_dest_filename,
+            "bind",
+            MsFlags::MS_BIND,
+            "",
+            &logger,
+        );
         assert!(result.is_ok(), "mount for test setup failed");
 
         let tests = &[

src/agent/src/namespace.rs

@@ -104,10 +104,7 @@ impl Namespace {
             if let Err(err) = || -> Result<()> {
                 let origin_ns_path = get_current_thread_ns_path(ns_type.get());
 
-                let source = Path::new(&origin_ns_path);
-                let destination = new_ns_path.as_path();
-
-                File::open(&source)?;
+                File::open(Path::new(&origin_ns_path))?;
 
                 // Create a new netns on the current thread.
                 let cf = ns_type.get_flags();
@@ -118,6 +115,8 @@ impl Namespace {
                     nix::unistd::sethostname(hostname.unwrap())?;
                 }
                 // Bind mount the new namespace from the current thread onto the mount point to persist it.
+                let source: &str = origin_ns_path.as_str();
+                let destination: &str = new_ns_path.as_path().to_str().unwrap_or("none");
 
                 let mut flags = MsFlags::empty();
 
@@ -132,7 +131,7 @@ impl Namespace {
 
                 baremount(source, destination, "none", flags, "", &logger).map_err(|e| {
                     anyhow!(
-                        "Failed to mount {:?} to {:?} with err:{:?}",
+                        "Failed to mount {} to {} with err:{:?}",
                         source,
                         destination,
                         e
@@ -251,126 +250,4 @@ mod tests {
         assert_eq!("pid", pid.get());
         assert_eq!(CloneFlags::CLONE_NEWPID, pid.get_flags());
     }
-
-    #[test]
-    fn test_new() {
-        // Create dummy logger and temp folder.
-        let logger = slog::Logger::root(slog::Discard, o!());
-
-        let ns_ipc = Namespace::new(&logger);
-        assert_eq!(NamespaceType::Ipc, ns_ipc.ns_type);
-    }
-
-    #[test]
-    fn test_get_ipc() {
-        // Create dummy logger and temp folder.
-        let logger = slog::Logger::root(slog::Discard, o!());
-
-        let ns_ipc = Namespace::new(&logger).get_ipc();
-        assert_eq!(NamespaceType::Ipc, ns_ipc.ns_type);
-    }
-
-    #[test]
-    fn test_get_uts_with_hostname() {
-        let hostname = String::from("a.test.com");
-        // Create dummy logger and temp folder.
-        let logger = slog::Logger::root(slog::Discard, o!());
-
-        let ns_uts = Namespace::new(&logger).get_uts(hostname.as_str());
-        assert_eq!(NamespaceType::Uts, ns_uts.ns_type);
-        assert!(ns_uts.hostname.is_some());
-    }
-
-    #[test]
-    fn test_get_uts() {
-        let hostname = String::from("");
-        // Create dummy logger and temp folder.
-        let logger = slog::Logger::root(slog::Discard, o!());
-
-        let ns_uts = Namespace::new(&logger).get_uts(hostname.as_str());
-        assert_eq!(NamespaceType::Uts, ns_uts.ns_type);
-        assert!(ns_uts.hostname.is_none());
-    }
-
-    #[test]
-    fn test_get_pid() {
-        // Create dummy logger and temp folder.
-        let logger = slog::Logger::root(slog::Discard, o!());
-
-        let ns_pid = Namespace::new(&logger).get_pid();
-        assert_eq!(NamespaceType::Pid, ns_pid.ns_type);
-    }
-
-    #[test]
-    fn test_set_root_dir() {
-        // Create dummy logger and temp folder.
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let tmpdir = Builder::new().prefix("pid").tempdir().unwrap();
-
-        let ns_root = Namespace::new(&logger).set_root_dir(tmpdir.path().to_str().unwrap());
-        assert_eq!(NamespaceType::Ipc, ns_root.ns_type);
-        assert_eq!(ns_root.persistent_ns_dir, tmpdir.path().to_str().unwrap());
-    }
-
-    #[test]
-    fn test_namespace_type_get() {
-        #[derive(Debug)]
-        struct TestData<'a> {
-            ns_type: NamespaceType,
-            str: &'a str,
-        }
-
-        let tests = &[
-            TestData {
-                ns_type: NamespaceType::Ipc,
-                str: "ipc",
-            },
-            TestData {
-                ns_type: NamespaceType::Uts,
-                str: "uts",
-            },
-            TestData {
-                ns_type: NamespaceType::Pid,
-                str: "pid",
-            },
-        ];
-
-        // Run the tests
-        for (i, d) in tests.iter().enumerate() {
-            // Create a string containing details of the test
-            let msg = format!("test[{}]: {:?}", i, d);
-            assert_eq!(d.str, d.ns_type.get(), "{}", msg)
-        }
-    }
-
-    #[test]
-    fn test_namespace_type_get_flags() {
-        #[derive(Debug)]
-        struct TestData {
-            ns_type: NamespaceType,
-            ns_flag: CloneFlags,
-        }
-
-        let tests = &[
-            TestData {
-                ns_type: NamespaceType::Ipc,
-                ns_flag: CloneFlags::CLONE_NEWIPC,
-            },
-            TestData {
-                ns_type: NamespaceType::Uts,
-                ns_flag: CloneFlags::CLONE_NEWUTS,
-            },
-            TestData {
-                ns_type: NamespaceType::Pid,
-                ns_flag: CloneFlags::CLONE_NEWPID,
-            },
-        ];
-
-        // Run the tests
-        for (i, d) in tests.iter().enumerate() {
-            // Create a string containing details of the test
-            let msg = format!("test[{}]: {:?}", i, d);
-            assert_eq!(d.ns_flag, d.ns_type.get_flags(), "{}", msg)
-        }
-    }
 }

src/agent/src/pci.rs

@@ -20,7 +20,7 @@ const FUNCTION_MAX: u8 = (1 << FUNCTION_BITS) - 1;
 
 // Represents a PCI function's slot (a.k.a. device) and function
 // numbers, giving its location on a single logical bus
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub struct SlotFn(u8);
 
 impl SlotFn {
@@ -94,7 +94,7 @@ impl fmt::Display for SlotFn {
     }
 }
 
-#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub struct Address {
     domain: u16,
     bus: u8,

src/agent/src/rpc.rs

@@ -111,18 +111,11 @@ pub struct AgentService {
 //     ^[a-zA-Z0-9][a-zA-Z0-9_.-]+$
 //
 fn verify_cid(id: &str) -> Result<()> {
-    let mut chars = id.chars();
-
-    let valid = match chars.next() {
-        Some(first)
-            if first.is_alphanumeric()
-                && id.len() > 1
-                && chars.all(|c| c.is_alphanumeric() || ['.', '-', '_'].contains(&c)) =>
-        {
-            true
-        }
-        _ => false,
-    };
+    let valid = id.len() > 1
+        && id.chars().next().unwrap().is_alphanumeric()
+        && id
+            .chars()
+            .all(|c| (c.is_alphanumeric() || ['.', '-', '_'].contains(&c)));
 
     match valid {
         true => Ok(()),
@@ -155,6 +148,10 @@ impl AgentService {
         };
 
         info!(sl!(), "receive createcontainer, spec: {:?}", &oci);
+        info!(
+            sl!(),
+            "receive createcontainer, storages: {:?}", &req.storages
+        );
 
         // Some devices need some extra processing (the ones invoked with
         // --device for instance), and that's what this call is doing. It
@@ -170,7 +167,13 @@ impl AgentService {
         // After all those storages have been processed, no matter the order
         // here, the agent will rely on rustjail (using the oci.Mounts
         // list) to bind mount all of them inside the container.
-        let m = add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone()).await?;
+        let m = add_storages(
+            sl!(),
+            req.storages.to_vec(),
+            self.sandbox.clone(),
+            Some(req.container_id.clone()),
+        )
+        .await?;
         {
             sandbox = self.sandbox.clone();
             s = sandbox.lock().await;
@@ -183,7 +186,7 @@ impl AgentService {
         update_device_cgroup(&mut oci)?;
 
         // Append guest hooks
-        append_guest_hooks(&s, &mut oci)?;
+        append_guest_hooks(&s, &mut oci);
 
         // write spec to bundle path, hooks might
         // read ocispec
@@ -205,14 +208,21 @@ impl AgentService {
             LinuxContainer::new(cid.as_str(), CONTAINER_BASE, opts, &sl!())?;
 
         let pipe_size = AGENT_CONFIG.read().await.container_pipe_size;
-
-        let p = if let Some(p) = oci.process {
-            Process::new(&sl!(), &p, cid.as_str(), true, pipe_size)?
+        let p = if oci.process.is_some() {
+            Process::new(
+                &sl!(),
+                oci.process.as_ref().unwrap(),
+                cid.as_str(),
+                true,
+                pipe_size,
+            )?
         } else {
             info!(sl!(), "no process configurations!");
             return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)));
         };
+
         ctr.start(p).await?;
+
         s.update_shared_pidns(&ctr)?;
         s.add_container(ctr);
         info!(sl!(), "created container!");
@@ -234,17 +244,11 @@ impl AgentService {
 
         ctr.exec()?;
 
-        if sid == cid {
-            return Ok(());
-        }
-
         // start oom event loop
-        if let Some(ref ctr) = ctr.cgroup_manager {
-            let cg_path = ctr.get_cg_path("memory");
-
-            if let Some(cg_path) = cg_path {
-                let rx = notifier::notify_oom(cid.as_str(), cg_path.to_string()).await?;
-
+        if sid != cid && ctr.cgroup_manager.is_some() {
+            let cg_path = ctr.cgroup_manager.as_ref().unwrap().get_cg_path("memory");
+            if cg_path.is_some() {
+                let rx = notifier::notify_oom(cid.as_str(), cg_path.unwrap()).await?;
                 s.run_oom_event_monitor(rx, cid.clone()).await;
             }
         }
@@ -344,13 +348,14 @@ impl AgentService {
         let s = self.sandbox.clone();
         let mut sandbox = s.lock().await;
 
-        let process = req
-            .process
-            .into_option()
-            .ok_or_else(|| anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)))?;
+        let process = if req.process.is_some() {
+            req.process.as_ref().unwrap()
+        } else {
+            return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)));
+        };
 
         let pipe_size = AGENT_CONFIG.read().await.container_pipe_size;
-        let ocip = rustjail::process_grpc_to_oci(&process);
+        let ocip = rustjail::process_grpc_to_oci(process);
         let p = Process::new(&sl!(), &ocip, exec_id.as_str(), false, pipe_size)?;
 
         let ctr = sandbox
@@ -368,6 +373,7 @@ impl AgentService {
         let eid = req.exec_id.clone();
         let s = self.sandbox.clone();
         let mut sandbox = s.lock().await;
+        let mut init = false;
 
         info!(
             sl!(),
@@ -376,14 +382,13 @@ impl AgentService {
             "exec-id" => eid.clone(),
         );
 
-        let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
+        if eid.is_empty() {
+            init = true;
+        }
 
-        let mut signal = Signal::try_from(req.signal as i32).map_err(|e| {
-            anyhow!(e).context(format!(
-                "failed to convert {:?} to signal (container-id: {}, exec-id: {})",
-                req.signal, cid, eid
-            ))
-        })?;
+        let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), init)?;
+
+        let mut signal = Signal::try_from(req.signal as i32).unwrap();
 
         // For container initProcess, if it hasn't installed handler for "SIGTERM" signal,
         // it will ignore the "SIGTERM" signal sent to it, thus send it "SIGKILL" signal
@@ -419,7 +424,7 @@ impl AgentService {
 
         let exit_rx = {
             let mut sandbox = s.lock().await;
-            let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
+            let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?;
 
             p.exit_watchers.push(exit_send);
             pid = p.pid;
@@ -442,11 +447,7 @@ impl AgentService {
             Some(p) => p,
             None => {
                 // Lost race, pick up exit code from channel
-                resp.status = exit_recv
-                    .recv()
-                    .await
-                    .ok_or_else(|| anyhow!("Failed to receive exit code"))?;
-
+                resp.status = exit_recv.recv().await.unwrap();
                 return Ok(resp);
             }
         };
@@ -477,7 +478,7 @@ impl AgentService {
         let writer = {
             let s = self.sandbox.clone();
             let mut sandbox = s.lock().await;
-            let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
+            let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?;
 
             // use ptmx io
             if p.term_master.is_some() {
@@ -488,7 +489,7 @@ impl AgentService {
             }
         };
 
-        let writer = writer.ok_or_else(|| anyhow!("cannot get writer"))?;
+        let writer = writer.unwrap();
         writer.lock().await.write_all(req.data.as_slice()).await?;
 
         let mut resp = WriteStreamResponse::new();
@@ -510,7 +511,7 @@ impl AgentService {
             let s = self.sandbox.clone();
             let mut sandbox = s.lock().await;
 
-            let p = sandbox.find_container_process(cid.as_str(), eid.as_str())?;
+            let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false)?;
 
             if p.term_master.is_some() {
                 term_exit_notifier = p.term_exit_notifier.clone();
@@ -530,7 +531,7 @@ impl AgentService {
             return Err(anyhow!(nix::Error::from_errno(nix::errno::Errno::EINVAL)));
         }
 
-        let reader = reader.ok_or_else(|| anyhow!("cannot get stream reader"))?;
+        let reader = reader.unwrap();
 
         tokio::select! {
             _ = term_exit_notifier.notified() => {
@@ -582,6 +583,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
     ) -> ttrpc::Result<Empty> {
         trace_rpc_call!(ctx, "remove_container", req);
         is_allowed!(req);
+
         match self.do_remove_container(req).await {
             Err(e) => Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string())),
             Ok(_) => Ok(Empty::new()),
@@ -648,8 +650,8 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
 
         let resp = Empty::new();
 
-        if let Some(res) = res.as_ref() {
-            let oci_res = rustjail::resources_grpc_to_oci(res);
+        if res.is_some() {
+            let oci_res = rustjail::resources_grpc_to_oci(&res.unwrap());
             match ctr.set(oci_res) {
                 Err(e) => {
                     return Err(ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()));
@@ -778,14 +780,12 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
 
-        let p = sandbox
-            .find_container_process(cid.as_str(), eid.as_str())
-            .map_err(|e| {
-                ttrpc_error(
-                    ttrpc::Code::INVALID_ARGUMENT,
-                    format!("invalid argument: {:?}", e),
-                )
-            })?;
+        let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| {
+            ttrpc_error(
+                ttrpc::Code::INVALID_ARGUMENT,
+                format!("invalid argument: {:?}", e),
+            )
+        })?;
 
         p.close_stdin();
 
@@ -804,33 +804,32 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         let eid = req.exec_id.clone();
         let s = Arc::clone(&self.sandbox);
         let mut sandbox = s.lock().await;
-        let p = sandbox
-            .find_container_process(cid.as_str(), eid.as_str())
-            .map_err(|e| {
-                ttrpc_error(
-                    ttrpc::Code::UNAVAILABLE,
-                    format!("invalid argument: {:?}", e),
-                )
-            })?;
+        let p = find_process(&mut sandbox, cid.as_str(), eid.as_str(), false).map_err(|e| {
+            ttrpc_error(
+                ttrpc::Code::UNAVAILABLE,
+                format!("invalid argument: {:?}", e),
+            )
+        })?;
 
-        if let Some(fd) = p.term_master {
-            unsafe {
-                let win = winsize {
-                    ws_row: req.row as c_ushort,
-                    ws_col: req.column as c_ushort,
-                    ws_xpixel: 0,
-                    ws_ypixel: 0,
-                };
-
-                let err = libc::ioctl(fd, TIOCSWINSZ, &win);
-                Errno::result(err).map(drop).map_err(|e| {
-                    ttrpc_error(ttrpc::Code::INTERNAL, format!("ioctl error: {:?}", e))
-                })?;
-            }
-        } else {
+        if p.term_master.is_none() {
             return Err(ttrpc_error(ttrpc::Code::UNAVAILABLE, "no tty".to_string()));
         }
 
+        let fd = p.term_master.unwrap();
+        unsafe {
+            let win = winsize {
+                ws_row: req.row as c_ushort,
+                ws_col: req.column as c_ushort,
+                ws_xpixel: 0,
+                ws_ypixel: 0,
+            };
+
+            let err = libc::ioctl(fd, TIOCSWINSZ, &win);
+            Errno::result(err)
+                .map(drop)
+                .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, format!("ioctl error: {:?}", e)))?;
+        }
+
         Ok(Empty::new())
     }
 
@@ -954,6 +953,25 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         })
     }
 
+    async fn start_tracing(
+        &self,
+        _ctx: &TtrpcContext,
+        req: protocols::agent::StartTracingRequest,
+    ) -> ttrpc::Result<Empty> {
+        info!(sl!(), "start_tracing {:?}", req);
+        is_allowed!(req);
+        Ok(Empty::new())
+    }
+
+    async fn stop_tracing(
+        &self,
+        _ctx: &TtrpcContext,
+        req: protocols::agent::StopTracingRequest,
+    ) -> ttrpc::Result<Empty> {
+        is_allowed!(req);
+        Ok(Empty::new())
+    }
+
     async fn create_sandbox(
         &self,
         ctx: &TtrpcContext,
@@ -995,7 +1013,7 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
                 .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
         }
 
-        match add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone()).await {
+        match add_storages(sl!(), req.storages.to_vec(), self.sandbox.clone(), None).await {
             Ok(m) => {
                 let sandbox = self.sandbox.clone();
                 let mut s = sandbox.lock().await;
@@ -1032,25 +1050,12 @@ impl protocols::agent_ttrpc::AgentService for AgentService {
         let mut sandbox = s.lock().await;
         // destroy all containers, clean up, notify agent to exit
         // etc.
-        sandbox
-            .destroy()
-            .await
-            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
+        sandbox.destroy().await.unwrap();
         // Close get_oom_event connection,
         // otherwise it will block the shutdown of ttrpc.
         sandbox.event_tx.take();
 
-        sandbox
-            .sender
-            .take()
-            .ok_or_else(|| {
-                ttrpc_error(
-                    ttrpc::Code::INTERNAL,
-                    "failed to get sandbox sender channel".to_string(),
-                )
-            })?
-            .send(1)
-            .map_err(|e| ttrpc_error(ttrpc::Code::INTERNAL, e.to_string()))?;
+        sandbox.sender.take().unwrap().send(1).unwrap();
 
         Ok(Empty::new())
     }
@@ -1309,7 +1314,11 @@ fn get_memory_info(block_size: bool, hotplug: bool) -> Result<(u64, bool)> {
         match stat::stat(SYSFS_MEMORY_HOTPLUG_PROBE_PATH) {
             Ok(_) => plug = true,
             Err(e) => {
-                info!(sl!(), "hotplug memory error: {:?}", e);
+                info!(
+                    sl!(),
+                    "hotplug memory error: {}",
+                    e.as_errno().unwrap().desc()
+                );
                 match e {
                     nix::Error::Sys(errno) => match errno {
                         Errno::ENOENT => plug = false,
@@ -1365,7 +1374,27 @@ async fn read_stream(reader: Arc<Mutex<ReadHalf<PipeStream>>>, l: usize) -> Resu
     Ok(content)
 }
 
-pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> Result<TtrpcServer> {
+fn find_process<'a>(
+    sandbox: &'a mut Sandbox,
+    cid: &'a str,
+    eid: &'a str,
+    init: bool,
+) -> Result<&'a mut Process> {
+    let ctr = sandbox
+        .get_container(cid)
+        .ok_or_else(|| anyhow!("Invalid container id"))?;
+
+    if init || eid.is_empty() {
+        return ctr
+            .processes
+            .get_mut(&ctr.init_process_pid)
+            .ok_or_else(|| anyhow!("cannot find init process!"));
+    }
+
+    ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id"))
+}
+
+pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> TtrpcServer {
     let agent_service = Box::new(AgentService { sandbox: s })
         as Box<dyn protocols::agent_ttrpc::AgentService + Send + Sync>;
 
@@ -1380,13 +1409,14 @@ pub fn start(s: Arc<Mutex<Sandbox>>, server_address: &str) -> Result<TtrpcServer
     let hservice = protocols::health_ttrpc::create_health(health_worker);
 
     let server = TtrpcServer::new()
-        .bind(server_address)?
+        .bind(server_address)
+        .unwrap()
         .register_service(aservice)
         .register_service(hservice);
 
     info!(sl!(), "ttRPC server started"; "address" => server_address);
 
-    Ok(server)
+    server
 }
 
 // This function updates the container namespaces configuration based on the
@@ -1431,28 +1461,24 @@ fn update_container_namespaces(
     // the create_sandbox request or create_container request.
     // Else set this to empty string so that a new pid namespace is
     // created for the container.
-    if sandbox_pidns {
-        if let Some(ref pidns) = &sandbox.sandbox_pidns {
-            pid_ns.path = String::from(pidns.path.as_str());
-        } else {
-            return Err(anyhow!("failed to get sandbox pidns"));
-        }
+    if sandbox_pidns && sandbox.sandbox_pidns.is_some() {
+        pid_ns.path = String::from(sandbox.sandbox_pidns.as_ref().unwrap().path.as_str());
     }
 
     linux.namespaces.push(pid_ns);
     Ok(())
 }
 
-fn append_guest_hooks(s: &Sandbox, oci: &mut Spec) -> Result<()> {
-    if let Some(ref guest_hooks) = s.hooks {
-        let mut hooks = oci.hooks.take().unwrap_or_default();
-        hooks.prestart.append(&mut guest_hooks.prestart.clone());
-        hooks.poststart.append(&mut guest_hooks.poststart.clone());
-        hooks.poststop.append(&mut guest_hooks.poststop.clone());
-        oci.hooks = Some(hooks);
+fn append_guest_hooks(s: &Sandbox, oci: &mut Spec) {
+    if s.hooks.is_none() {
+        return;
     }
-
-    Ok(())
+    let guest_hooks = s.hooks.as_ref().unwrap();
+    let mut hooks = oci.hooks.take().unwrap_or_default();
+    hooks.prestart.append(&mut guest_hooks.prestart.clone());
+    hooks.poststart.append(&mut guest_hooks.poststart.clone());
+    hooks.poststop.append(&mut guest_hooks.poststop.clone());
+    oci.hooks = Some(hooks);
 }
 
 // Check is the container process installed the
@@ -1542,7 +1568,7 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         PathBuf::from("/")
     };
 
-    fs::create_dir_all(&dir).or_else(|e| {
+    fs::create_dir_all(dir.to_str().unwrap()).or_else(|e| {
         if e.kind() != std::io::ErrorKind::AlreadyExists {
             return Err(e);
         }
@@ -1550,7 +1576,10 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         Ok(())
     })?;
 
-    std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(req.dir_mode))?;
+    std::fs::set_permissions(
+        dir.to_str().unwrap(),
+        std::fs::Permissions::from_mode(req.dir_mode),
+    )?;
 
     let mut tmpfile = path.clone();
     tmpfile.set_extension("tmp");
@@ -1559,10 +1588,10 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
         .write(true)
         .create(true)
         .truncate(false)
-        .open(&tmpfile)?;
+        .open(tmpfile.to_str().unwrap())?;
 
     file.write_all_at(req.data.as_slice(), req.offset as u64)?;
-    let st = stat::stat(&tmpfile)?;
+    let st = stat::stat(tmpfile.to_str().unwrap())?;
 
     if st.st_size != req.file_size {
         return Ok(());
@@ -1571,7 +1600,7 @@ fn do_copy_file(req: &CopyFileRequest) -> Result<()> {
     file.set_permissions(std::fs::Permissions::from_mode(req.file_mode))?;
 
     unistd::chown(
-        &tmpfile,
+        tmpfile.to_str().unwrap(),
         Some(Uid::from_raw(req.uid as u32)),
         Some(Gid::from_raw(req.gid as u32)),
     )?;
@@ -1608,13 +1637,10 @@ async fn do_add_swap(sandbox: &Arc<Mutex<Sandbox>>, req: &AddSwapRequest) -> Res
 // - container rootfs bind mounted at /<CONTAINER_BASE>/<cid>/rootfs
 // - modify container spec root to point to /<CONTAINER_BASE>/<cid>/rootfs
 fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
-    let spec_root = if let Some(sr) = &spec.root {
-        sr
-    } else {
+    if spec.root.is_none() {
         return Err(nix::Error::Sys(Errno::EINVAL).into());
-    };
-
-    let spec_root_path = Path::new(&spec_root.path);
+    }
+    let spec_root = spec.root.as_ref().unwrap();
 
     let bundle_path = Path::new(CONTAINER_BASE).join(cid);
     let config_path = bundle_path.join("config.json");
@@ -1622,36 +1648,22 @@ fn setup_bundle(cid: &str, spec: &mut Spec) -> Result<PathBuf> {
 
     fs::create_dir_all(&rootfs_path)?;
     baremount(
-        spec_root_path,
-        &rootfs_path,
+        &spec_root.path,
+        rootfs_path.to_str().unwrap(),
         "bind",
         MsFlags::MS_BIND,
         "",
         &sl!(),
     )?;
-
-    let rootfs_path_name = rootfs_path
-        .to_str()
-        .ok_or_else(|| anyhow!("failed to convert rootfs to unicode"))?
-        .to_string();
-
     spec.root = Some(Root {
-        path: rootfs_path_name,
+        path: rootfs_path.to_str().unwrap().to_owned(),
         readonly: spec_root.readonly,
     });
 
-    let _ = spec.save(
-        config_path
-            .to_str()
-            .ok_or_else(|| anyhow!("cannot convert path to unicode"))?,
-    );
+    let _ = spec.save(config_path.to_str().unwrap());
 
     let olddir = unistd::getcwd().context("cannot getcwd")?;
-    unistd::chdir(
-        bundle_path
-            .to_str()
-            .ok_or_else(|| anyhow!("cannot convert bundle path to unicode"))?,
-    )?;
+    unistd::chdir(bundle_path.to_str().unwrap())?;
 
     Ok(olddir)
 }
@@ -1684,8 +1696,8 @@ fn load_kernel_module(module: &protocols::agent::KernelModule) -> Result<()> {
 
     match status.code() {
         Some(code) => {
-            let std_out = String::from_utf8_lossy(&output.stdout);
-            let std_err = String::from_utf8_lossy(&output.stderr);
+            let std_out: String = String::from_utf8(output.stdout).unwrap();
+            let std_err: String = String::from_utf8(output.stderr).unwrap();
             let msg = format!(
                 "load_kernel_module return code: {} stdout:{} stderr:{}",
                 code, std_out, std_err
@@ -1708,6 +1720,7 @@ mod tests {
             fd: -1,
             mh: MessageHeader::default(),
             metadata: std::collections::HashMap::new(),
+            timeout_nano: 0,
         }
     }
 
@@ -1748,7 +1761,7 @@ mod tests {
         let mut oci = Spec {
             ..Default::default()
         };
-        append_guest_hooks(&s, &mut oci).unwrap();
+        append_guest_hooks(&s, &mut oci);
         assert_eq!(s.hooks, oci.hooks);
     }
 

src/agent/src/sandbox.rs

@@ -226,21 +226,6 @@ impl Sandbox {
         None
     }
 
-    pub fn find_container_process(&mut self, cid: &str, eid: &str) -> Result<&mut Process> {
-        let ctr = self
-            .get_container(cid)
-            .ok_or_else(|| anyhow!("Invalid container id"))?;
-
-        if eid.is_empty() {
-            return ctr
-                .processes
-                .get_mut(&ctr.init_process_pid)
-                .ok_or_else(|| anyhow!("cannot find init process!"));
-        }
-
-        ctr.get_process(eid).map_err(|_| anyhow!("Invalid exec id"))
-    }
-
     #[instrument]
     pub async fn destroy(&mut self) -> Result<()> {
         for ctr in self.containers.values_mut() {
@@ -465,23 +450,18 @@ fn online_memory(logger: &Logger) -> Result<()> {
 mod tests {
     use super::Sandbox;
     use crate::{mount::baremount, skip_if_not_root};
-    use anyhow::{anyhow, Error};
+    use anyhow::Error;
     use nix::mount::MsFlags;
     use oci::{Linux, Root, Spec};
     use rustjail::container::LinuxContainer;
-    use rustjail::process::Process;
     use rustjail::specconv::CreateOpts;
     use slog::Logger;
     use std::fs::{self, File};
     use std::os::unix::fs::PermissionsExt;
-    use std::path::Path;
-    use tempfile::{tempdir, Builder, TempDir};
+    use tempfile::Builder;
 
     fn bind_mount(src: &str, dst: &str, logger: &Logger) -> Result<(), Error> {
-        let src_path = Path::new(src);
-        let dst_path = Path::new(dst);
-
-        baremount(src_path, dst_path, "bind", MsFlags::MS_BIND, "", logger)
+        baremount(src, dst, "bind", MsFlags::MS_BIND, "", logger)
     }
 
     use serial_test::serial;
@@ -704,31 +684,23 @@ mod tests {
         }
     }
 
-    fn create_linuxcontainer() -> (LinuxContainer, TempDir) {
-        // Create a temporal directory
-        let dir = tempdir()
-            .map_err(|e| anyhow!(e).context("tempdir failed"))
-            .unwrap();
-
-        // Create a new container
-        (
-            LinuxContainer::new(
-                "some_id",
-                dir.path().join("rootfs").to_str().unwrap(),
-                create_dummy_opts(),
-                &slog_scope::logger(),
-            )
-            .unwrap(),
-            dir,
+    fn create_linuxcontainer() -> LinuxContainer {
+        LinuxContainer::new(
+            "some_id",
+            "/run/agent",
+            create_dummy_opts(),
+            &slog_scope::logger(),
         )
+        .unwrap()
     }
 
     #[tokio::test]
     #[serial]
     async fn get_container_entry_exist() {
+        skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
-        let (linux_container, _root) = create_linuxcontainer();
+        let linux_container = create_linuxcontainer();
 
         s.containers
             .insert("testContainerID".to_string(), linux_container);
@@ -749,9 +721,10 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn add_and_get_container() {
+        skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
-        let (linux_container, _root) = create_linuxcontainer();
+        let linux_container = create_linuxcontainer();
 
         s.add_container(linux_container);
         assert!(s.get_container("some_id").is_some());
@@ -760,11 +733,12 @@ mod tests {
     #[tokio::test]
     #[serial]
     async fn update_shared_pidns() {
+        skip_if_not_root!();
         let logger = slog::Logger::root(slog::Discard, o!());
         let mut s = Sandbox::new(&logger).unwrap();
         let test_pid = 9999;
 
-        let (mut linux_container, _root) = create_linuxcontainer();
+        let mut linux_container = create_linuxcontainer();
         linux_container.init_process_pid = test_pid;
 
         s.update_shared_pidns(&linux_container).unwrap();
@@ -807,49 +781,4 @@ mod tests {
         let ret = s.destroy().await;
         assert!(ret.is_ok());
     }
-
-    #[tokio::test]
-    async fn test_find_container_process() {
-        let logger = slog::Logger::root(slog::Discard, o!());
-        let mut s = Sandbox::new(&logger).unwrap();
-        let cid = "container-123";
-
-        let (mut linux_container, _root) = create_linuxcontainer();
-        linux_container.init_process_pid = 1;
-        linux_container.id = cid.to_string();
-        // add init process
-        linux_container.processes.insert(
-            1,
-            Process::new(&logger, &oci::Process::default(), "1", true, 1).unwrap(),
-        );
-        // add exec process
-        linux_container.processes.insert(
-            123,
-            Process::new(&logger, &oci::Process::default(), "exec-123", false, 1).unwrap(),
-        );
-
-        s.add_container(linux_container);
-
-        // empty exec-id will return init process
-        let p = s.find_container_process(cid, "");
-        assert!(p.is_ok(), "Expecting Ok, Got {:?}", p);
-        let p = p.unwrap();
-        assert_eq!("1", p.exec_id, "exec_id should be 1");
-        assert!(p.init, "init flag should be true");
-
-        // get exist exec-id will return the exec process
-        let p = s.find_container_process(cid, "exec-123");
-        assert!(p.is_ok(), "Expecting Ok, Got {:?}", p);
-        let p = p.unwrap();
-        assert_eq!("exec-123", p.exec_id, "exec_id should be exec-123");
-        assert!(!p.init, "init flag should be false");
-
-        // get not exist exec-id will return error
-        let p = s.find_container_process(cid, "exec-456");
-        assert!(p.is_err(), "Expecting Error, Got {:?}", p);
-
-        // container does not exist
-        let p = s.find_container_process("not-exist-cid", "");
-        assert!(p.is_err(), "Expecting Error, Got {:?}", p);
-    }
 }

src/agent/src/util.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-use anyhow::{anyhow, Result};
+use anyhow::Result;
 use futures::StreamExt;
 use std::io;
 use std::io::ErrorKind;
@@ -64,12 +64,8 @@ pub fn get_vsock_incoming(fd: RawFd) -> Incoming {
 
 #[instrument]
 pub async fn get_vsock_stream(fd: RawFd) -> Result<VsockStream> {
-    let stream = get_vsock_incoming(fd)
-        .next()
-        .await
-        .ok_or_else(|| anyhow!("cannot handle incoming vsock connection"))?;
-
-    Ok(stream?)
+    let stream = get_vsock_incoming(fd).next().await.unwrap()?;
+    Ok(stream)
 }
 
 #[cfg(test)]
@@ -128,9 +124,7 @@ mod tests {
 
             let mut vec_locked = vec_ref.lock();
 
-            let v = vec_locked
-                .as_deref_mut()
-                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))?;
+            let v = vec_locked.as_deref_mut().unwrap();
 
             std::io::Write::flush(v)
         }

src/agent/src/watcher.rs

@@ -109,11 +109,12 @@ impl Storage {
 
         // if we are creating a directory: just create it, nothing more to do
         if source_file_path.symlink_metadata()?.file_type().is_dir() {
-            fs::create_dir_all(source_file_path)
+            let dest_file_path = self.make_target_path(&source_file_path)?;
+
+            fs::create_dir_all(&dest_file_path)
                 .await
-                .with_context(|| {
-                    format!("Unable to mkdir all for {}", source_file_path.display())
-                })?
+                .with_context(|| format!("Unable to mkdir all for {}", dest_file_path.display()))?;
+            return Ok(());
         }
 
         // Assume we are dealing with either a file or a symlink now:
@@ -366,8 +367,8 @@ impl SandboxStorages {
                         }
 
                         match baremount(
-                            entry.source_mount_point.as_path(),
-                            entry.target_mount_point.as_path(),
+                            entry.source_mount_point.to_str().unwrap(),
+                            entry.target_mount_point.to_str().unwrap(),
                             "bind",
                             MsFlags::MS_BIND,
                             "bind",
@@ -477,8 +478,8 @@ impl BindWatcher {
         fs::create_dir_all(WATCH_MOUNT_POINT_PATH).await?;
 
         baremount(
-            Path::new("tmpfs"),
-            Path::new(WATCH_MOUNT_POINT_PATH),
+            "tmpfs",
+            WATCH_MOUNT_POINT_PATH,
             "tmpfs",
             MsFlags::empty(),
             "",
@@ -922,7 +923,7 @@ mod tests {
             .file_type()
             .is_symlink());
         assert_eq!(fs::read_link(&dst_symlink_file).unwrap(), src_file);
-        assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo")
+        assert_eq!(fs::read_to_string(&dst_symlink_file).unwrap(), "foo");
     }
 
     #[tokio::test]
@@ -1076,6 +1077,10 @@ mod tests {
         fs::create_dir_all(source_dir.path().join("A/B")).unwrap();
         fs::write(source_dir.path().join("A/B/1.txt"), "two").unwrap();
 
+        // A/C is an empty directory
+        let empty_dir = "A/C";
+        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+
         // delay 20 ms between writes to files in order to ensure filesystem timestamps are unique
         thread::sleep(Duration::from_millis(20));
 
@@ -1091,7 +1096,10 @@ mod tests {
 
         let logger = slog::Logger::root(slog::Discard, o!());
 
-        assert_eq!(entry.scan(&logger).await.unwrap(), 5);
+        assert_eq!(entry.scan(&logger).await.unwrap(), 6);
+
+        // check empty directory
+        assert!(dest_dir.path().join(empty_dir).exists());
 
         // Should copy no files since nothing is changed since last check
         assert_eq!(entry.scan(&logger).await.unwrap(), 0);
@@ -1112,6 +1120,12 @@ mod tests {
         // Update another file
         fs::write(source_dir.path().join("1.txt"), "updated").unwrap();
         assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+
+        // create another empty directory A/C/D
+        let empty_dir = "A/C/D";
+        fs::create_dir_all(source_dir.path().join(empty_dir)).unwrap();
+        assert_eq!(entry.scan(&logger).await.unwrap(), 1);
+        assert!(dest_dir.path().join(empty_dir).exists());
     }
 
     #[tokio::test]

src/runtime/.gitignore

@@ -11,6 +11,7 @@ config-generated.go
 /pkg/containerd-shim-v2/monitor_address
 /data/kata-collect-data.sh
 /kata-monitor
+/kata-netmon
 /kata-runtime
 /pkg/katautils/config-settings.go
 /virtcontainers/hack/virtc/virtc

src/runtime/Makefile

@@ -55,6 +55,11 @@ RUNTIME_OUTPUT = $(CURDIR)/$(TARGET)
 RUNTIME_DIR = $(CLI_DIR)/$(TARGET)
 BINLIST += $(TARGET)
 
+NETMON_DIR = $(CLI_DIR)/netmon
+NETMON_TARGET = $(PROJECT_TYPE)-netmon
+NETMON_RUNTIME_OUTPUT = $(CURDIR)/$(NETMON_TARGET)
+BINLIBEXECLIST += $(NETMON_TARGET)
+
 DESTDIR ?= /
 
 ifeq ($(PREFIX),)
@@ -137,6 +142,9 @@ ACRNVALIDHYPERVISORPATHS := [\"$(ACRNPATH)\"]
 ACRNCTLPATH := $(ACRNBINDIR)/$(ACRNCTLCMD)
 ACRNVALIDCTLPATHS := [\"$(ACRNCTLPATH)\"]
 
+NETMONCMD := $(BIN_PREFIX)-netmon
+NETMONPATH := $(PKGLIBEXECDIR)/$(NETMONCMD)
+
 # Default number of vCPUs
 DEFVCPUS := 1
 # Default maximum number of vCPUs
@@ -408,6 +416,7 @@ USER_VARS += PROJECT_PREFIX
 USER_VARS += PROJECT_TAG
 USER_VARS += PROJECT_TYPE
 USER_VARS += PROJECT_URL
+USER_VARS += NETMONPATH
 USER_VARS += QEMUBINDIR
 USER_VARS += QEMUCMD
 USER_VARS += QEMUPATH
@@ -500,7 +509,7 @@ define SHOW_ARCH
   $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
 endef
 
-all: runtime containerd-shim-v2 monitor
+all: runtime containerd-shim-v2 netmon monitor
 
 # Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
 # git revision string.  They will only be rebuilt if the revision string
@@ -516,6 +525,11 @@ containerd-shim-v2: $(SHIMV2_OUTPUT)
 
 monitor: $(MONITOR_OUTPUT)
 
+netmon: $(NETMON_RUNTIME_OUTPUT)
+
+$(NETMON_RUNTIME_OUTPUT): $(SOURCES) VERSION
+	$(QUIET_BUILD)(cd $(NETMON_DIR) && go build $(BUILDFLAGS) -o $@ -ldflags "-X main.version=$(VERSION)" $(KATA_LDFLAGS))
+
 runtime: $(RUNTIME_OUTPUT) $(CONFIGS)
 .DEFAULT: default
 
@@ -624,13 +638,15 @@ coverage:
 	go test -v -mod=vendor -covermode=atomic -coverprofile=coverage.txt ./...
 	go tool cover -html=coverage.txt -o coverage.html
 
-install: all install-runtime install-containerd-shim-v2 install-monitor
+install: all install-runtime install-containerd-shim-v2 install-monitor install-netmon
 
 install-bin: $(BINLIST)
 	$(QUIET_INST)$(foreach f,$(BINLIST),$(call INSTALL_EXEC,$f,$(BINDIR)))
 
 install-runtime: runtime install-scripts install-completions install-configs install-bin
 
+install-netmon: install-bin-libexec
+
 install-containerd-shim-v2: $(SHIMV2)
 	$(QUIET_INST)$(call INSTALL_EXEC,$<,$(BINDIR))
 
@@ -662,6 +678,7 @@ clean:
 	$(QUIET_CLEAN)rm -f \
 		$(CONFIGS) \
 		$(GENERATED_FILES) \
+		$(NETMON_TARGET) \
 		$(MONITOR) \
 		$(SHIMV2) \
 		$(TARGET) \
@@ -689,7 +706,9 @@ show-usage: show-header
 	@printf "\tgenerate-config            : create configuration file.\n"
 	@printf "\tinstall                    : install everything.\n"
 	@printf "\tinstall-containerd-shim-v2 : only install containerd shim v2 files.\n"
+	@printf "\tinstall-netmon             : only install netmon files.\n"
 	@printf "\tinstall-runtime            : only install runtime files.\n"
+	@printf "\tnetmon                     : only build netmon.\n"
 	@printf "\truntime                    : only build runtime.\n"
 	@printf "\tshow-arches                : show supported architectures (ARCH variable values).\n"
 	@printf "\tshow-summary               : show install locations.\n"

src/runtime/cmd/kata-runtime/factory.go

@@ -16,10 +16,10 @@ import (
 	"time"
 
 	"github.com/gogo/protobuf/types"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	pb "github.com/kata-containers/kata-containers/src/runtime/protocols/cache"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vf "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 	"golang.org/x/sys/unix"

src/runtime/cmd/kata-runtime/kata-check.go

@@ -26,8 +26,8 @@ import (
 	"syscall"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli"
 )

src/runtime/cmd/kata-runtime/kata-check_generic_test.go

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-//go:build arm64 || ppc64le
 // +build arm64 ppc64le
 
 package main

src/runtime/cmd/kata-runtime/kata-env.go

@@ -18,10 +18,10 @@ import (
 	"github.com/urfave/cli"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	vcUtils "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -140,6 +140,14 @@ type HostInfo struct {
 	SupportVSocks             bool
 }
 
+// NetmonInfo stores netmon details
+type NetmonInfo struct {
+	Path    string
+	Version VersionInfo
+	Debug   bool
+	Enable  bool
+}
+
 // EnvInfo collects all information that will be displayed by the
 // env command.
 //
@@ -151,6 +159,7 @@ type EnvInfo struct {
 	Initrd     InitrdInfo
 	Hypervisor HypervisorInfo
 	Runtime    RuntimeInfo
+	Netmon     NetmonInfo
 	Host       HostInfo
 	Agent      AgentInfo
 }
@@ -267,6 +276,26 @@ func getMemoryInfo() MemoryInfo {
 	}
 }
 
+func getNetmonInfo(config oci.RuntimeConfig) NetmonInfo {
+	netmonConfig := config.NetmonConfig
+
+	var netmonVersionInfo VersionInfo
+	if version, err := getCommandVersion(netmonConfig.Path); err != nil {
+		netmonVersionInfo = unknownVersionInfo
+	} else {
+		netmonVersionInfo = constructVersionInfo(version)
+	}
+
+	netmon := NetmonInfo{
+		Version: netmonVersionInfo,
+		Path:    netmonConfig.Path,
+		Debug:   netmonConfig.Debug,
+		Enable:  netmonConfig.Enable,
+	}
+
+	return netmon
+}
+
 func getCommandVersion(cmd string) (string, error) {
 	return utils.RunCommand([]string{cmd, "--version"})
 }
@@ -335,6 +364,8 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e
 		return EnvInfo{}, err
 	}
 
+	netmon := getNetmonInfo(config)
+
 	agent, err := getAgentInfo(config)
 	if err != nil {
 		return EnvInfo{}, err
@@ -367,6 +398,7 @@ func getEnvInfo(configFile string, config oci.RuntimeConfig) (env EnvInfo, err e
 		Initrd:     initrd,
 		Agent:      agent,
 		Host:       host,
+		Netmon:     netmon,
 	}
 
 	return env, nil

src/runtime/cmd/kata-runtime/kata-env_generic_test.go

@@ -3,7 +3,6 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-//go:build arm64 || ppc64le
 // +build arm64 ppc64le
 
 package main

src/runtime/cmd/kata-runtime/kata-env_test.go

@@ -27,11 +27,12 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/stretchr/testify/assert"
 )
 
 var (
+	testNetmonVersion     = "netmon version 0.1"
 	testHypervisorVersion = "QEMU emulator version 2.7.0+git.741f430a96-6.1, Copyright (c) 2003-2016 Fabrice Bellard and the QEMU Project developers"
 )
 
@@ -40,6 +41,7 @@ var (
 	enableVirtioFS  = false
 	runtimeDebug    = false
 	runtimeTrace    = false
+	netmonDebug     = false
 	agentDebug      = false
 	agentTrace      = false
 )
@@ -81,6 +83,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	imagePath := filepath.Join(prefixDir, "image")
 	kernelParams := "foo=bar xyz"
 	machineType := "machineType"
+	netmonPath := filepath.Join(prefixDir, "netmon")
 	disableBlock := true
 	blockStorageDriver := "virtio-scsi"
 	enableIOThreads := true
@@ -104,6 +107,11 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		}
 	}
 
+	err = makeVersionBinary(netmonPath, testNetmonVersion)
+	if err != nil {
+		return "", oci.RuntimeConfig{}, err
+	}
+
 	err = makeVersionBinary(hypervisorPath, testHypervisorVersion)
 	if err != nil {
 		return "", oci.RuntimeConfig{}, err
@@ -122,6 +130,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		ImagePath:            imagePath,
 		KernelParams:         kernelParams,
 		MachineType:          machineType,
+		NetmonPath:           netmonPath,
 		LogPath:              logPath,
 		DefaultGuestHookPath: hypConfig.GuestHookPath,
 		DisableBlock:         disableBlock,
@@ -137,6 +146,7 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 		HypervisorDebug:      hypervisorDebug,
 		RuntimeDebug:         runtimeDebug,
 		RuntimeTrace:         runtimeTrace,
+		NetmonDebug:          netmonDebug,
 		AgentDebug:           agentDebug,
 		AgentTrace:           agentTrace,
 		SharedFS:             sharedFS,
@@ -159,6 +169,15 @@ func makeRuntimeConfig(prefixDir string) (configFile string, config oci.RuntimeC
 	return configFile, config, nil
 }
 
+func getExpectedNetmonDetails(config oci.RuntimeConfig) (NetmonInfo, error) {
+	return NetmonInfo{
+		Version: constructVersionInfo(testNetmonVersion),
+		Path:    config.NetmonConfig.Path,
+		Debug:   config.NetmonConfig.Debug,
+		Enable:  config.NetmonConfig.Enable,
+	}, nil
+}
+
 func getExpectedAgentDetails(config oci.RuntimeConfig) (AgentInfo, error) {
 
 	agentConfig := config.AgentConfig
@@ -333,6 +352,11 @@ func getExpectedSettings(config oci.RuntimeConfig, tmpdir, configFile string) (E
 		return EnvInfo{}, err
 	}
 
+	netmon, err := getExpectedNetmonDetails(config)
+	if err != nil {
+		return EnvInfo{}, err
+	}
+
 	hypervisor := getExpectedHypervisor(config)
 	kernel := getExpectedKernel(config)
 	image := getExpectedImage(config)
@@ -345,6 +369,7 @@ func getExpectedSettings(config oci.RuntimeConfig, tmpdir, configFile string) (E
 		Kernel:     kernel,
 		Agent:      agent,
 		Host:       host,
+		Netmon:     netmon,
 	}
 
 	return env, nil
@@ -587,6 +612,50 @@ func TestEnvGetRuntimeInfo(t *testing.T) {
 	assert.Equal(t, expectedRuntime, runtime)
 }
 
+func TestEnvGetNetmonInfo(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(t, err)
+
+	expectedNetmon, err := getExpectedNetmonDetails(config)
+	assert.NoError(t, err)
+
+	netmon := getNetmonInfo(config)
+	assert.NoError(t, err)
+
+	assert.Equal(t, expectedNetmon, netmon)
+}
+
+func TestEnvGetNetmonInfoNoVersion(t *testing.T) {
+	tmpdir, err := ioutil.TempDir("", "")
+	if err != nil {
+		panic(err)
+	}
+	defer os.RemoveAll(tmpdir)
+
+	_, config, err := makeRuntimeConfig(tmpdir)
+	assert.NoError(t, err)
+
+	expectedNetmon, err := getExpectedNetmonDetails(config)
+	assert.NoError(t, err)
+
+	// remove the netmon ensuring its version cannot be queried
+	err = os.Remove(config.NetmonConfig.Path)
+	assert.NoError(t, err)
+
+	expectedNetmon.Version = unknownVersionInfo
+
+	netmon := getNetmonInfo(config)
+	assert.NoError(t, err)
+
+	assert.Equal(t, expectedNetmon, netmon)
+}
+
 func TestEnvGetAgentInfo(t *testing.T) {
 	tmpdir, err := ioutil.TempDir("", "")
 	if err != nil {
@@ -983,13 +1052,11 @@ func TestGetHypervisorInfoSocket(t *testing.T) {
 		{vc.QemuHypervisor, false},
 	}
 
-	config.HypervisorConfig.VMStorePath = "/foo"
-	config.HypervisorConfig.RunStorePath = "/bar"
-
 	for i, details := range hypervisors {
 		msg := fmt.Sprintf("hypervisor[%d]: %+v", i, details)
 
 		config.HypervisorType = details.hType
+
 		info, err := getHypervisorInfo(config)
 		assert.NoError(err, msg)
 

src/runtime/cmd/kata-runtime/main.go

@@ -18,12 +18,12 @@ import (
 	"syscall"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/signals"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	vf "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory"
 	tl "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory/template"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"

src/runtime/cmd/kata-runtime/main_test.go

@@ -21,8 +21,8 @@ import (
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"

src/runtime/cmd/netmon/netmon.go

@@ -0,0 +1,683 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log/syslog"
+	"net"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/signals"
+	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
+
+	"github.com/sirupsen/logrus"
+	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	netmonName = "kata-netmon"
+
+	kataCmd              = "kata-network"
+	kataCLIAddIfaceCmd   = "add-iface"
+	kataCLIDelIfaceCmd   = "del-iface"
+	kataCLIUpdtRoutesCmd = "update-routes"
+
+	kataSuffix = "kata"
+
+	// sharedFile is the name of the file that will be used to share
+	// the data between this process and the kata-runtime process
+	// responsible for updating the network.
+	sharedFile      = "shared.json"
+	storageFilePerm = os.FileMode(0640)
+	storageDirPerm  = os.FileMode(0750)
+)
+
+var (
+	// version is the netmon version. This variable is populated at build time.
+	version = "unknown"
+
+	// For simplicity the code will only focus on IPv4 addresses for now.
+	netlinkFamily = netlink.FAMILY_ALL
+
+	storageParentPath = "/var/run/kata-containers/netmon/sbs"
+)
+
+type netmonParams struct {
+	sandboxID   string
+	runtimePath string
+	logLevel    string
+	debug       bool
+}
+
+type netmon struct {
+	netIfaces map[int]pbTypes.Interface
+
+	linkUpdateCh chan netlink.LinkUpdate
+	linkDoneCh   chan struct{}
+
+	rtUpdateCh chan netlink.RouteUpdate
+	rtDoneCh   chan struct{}
+
+	netHandler *netlink.Handle
+
+	storagePath string
+	sharedFile  string
+
+	netmonParams
+}
+
+var netmonLog = logrus.New()
+
+func printVersion() {
+	fmt.Printf("%s version %s\n", netmonName, version)
+}
+
+const componentDescription = `is a network monitoring process that is intended to be started in the
+appropriate network namespace so that it can listen to any event related to
+link and routes. Whenever a new interface or route is created/updated, it is
+responsible for calling into the kata-runtime CLI to ask for the actual
+creation/update of the given interface or route.
+`
+
+func printComponentDescription() {
+	fmt.Printf("\n%s %s\n", netmonName, componentDescription)
+}
+
+func parseOptions() netmonParams {
+	var version, help bool
+
+	params := netmonParams{}
+
+	flag.BoolVar(&help, "h", false, "describe component usage")
+	flag.BoolVar(&help, "help", false, "")
+	flag.BoolVar(&params.debug, "d", false, "enable debug mode")
+	flag.BoolVar(&version, "v", false, "display program version and exit")
+	flag.BoolVar(&version, "version", false, "")
+	flag.StringVar(&params.sandboxID, "s", "", "sandbox id (required)")
+	flag.StringVar(&params.runtimePath, "r", "", "runtime path (required)")
+	flag.StringVar(&params.logLevel, "log", "warn",
+		"log messages above specified level: debug, warn, error, fatal or panic")
+
+	flag.Parse()
+
+	if help {
+		printComponentDescription()
+		flag.PrintDefaults()
+		os.Exit(0)
+	}
+
+	if version {
+		printVersion()
+		os.Exit(0)
+	}
+
+	if params.sandboxID == "" {
+		fmt.Fprintf(os.Stderr, "Error: sandbox id is empty, one must be provided\n")
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	if params.runtimePath == "" {
+		fmt.Fprintf(os.Stderr, "Error: runtime path is empty, one must be provided\n")
+		flag.PrintDefaults()
+		os.Exit(1)
+	}
+
+	return params
+}
+
+func newNetmon(params netmonParams) (*netmon, error) {
+	handler, err := netlink.NewHandle(netlinkFamily)
+	if err != nil {
+		return nil, err
+	}
+
+	n := &netmon{
+		netmonParams: params,
+		storagePath:  filepath.Join(storageParentPath, params.sandboxID),
+		sharedFile:   filepath.Join(storageParentPath, params.sandboxID, sharedFile),
+		netIfaces:    make(map[int]pbTypes.Interface),
+		linkUpdateCh: make(chan netlink.LinkUpdate),
+		linkDoneCh:   make(chan struct{}),
+		rtUpdateCh:   make(chan netlink.RouteUpdate),
+		rtDoneCh:     make(chan struct{}),
+		netHandler:   handler,
+	}
+
+	if err := os.MkdirAll(n.storagePath, storageDirPerm); err != nil {
+		return nil, err
+	}
+
+	return n, nil
+}
+
+func (n *netmon) cleanup() {
+	os.RemoveAll(n.storagePath)
+	n.netHandler.Close()
+	close(n.linkDoneCh)
+	close(n.rtDoneCh)
+}
+
+// setupSignalHandler sets up signal handling, starting a go routine to deal
+// with signals as they arrive.
+func (n *netmon) setupSignalHandler() {
+	signals.SetLogger(n.logger())
+
+	sigCh := make(chan os.Signal, 8)
+
+	for _, sig := range signals.HandledSignals() {
+		signal.Notify(sigCh, sig)
+	}
+
+	go func() {
+		for {
+			sig := <-sigCh
+
+			nativeSignal, ok := sig.(syscall.Signal)
+			if !ok {
+				err := errors.New("unknown signal")
+				netmonLog.WithError(err).WithField("signal", sig.String()).Error()
+				continue
+			}
+
+			if signals.FatalSignal(nativeSignal) {
+				netmonLog.WithField("signal", sig).Error("received fatal signal")
+				signals.Die(nil)
+			} else if n.debug && signals.NonFatalSignal(nativeSignal) {
+				netmonLog.WithField("signal", sig).Debug("handling signal")
+				signals.Backtrace()
+			}
+		}
+	}()
+}
+
+func (n *netmon) logger() *logrus.Entry {
+	fields := logrus.Fields{
+		"name":   netmonName,
+		"pid":    os.Getpid(),
+		"source": "netmon",
+	}
+
+	if n.sandboxID != "" {
+		fields["sandbox"] = n.sandboxID
+	}
+
+	return netmonLog.WithFields(fields)
+}
+
+func (n *netmon) setupLogger() error {
+	level, err := logrus.ParseLevel(n.logLevel)
+	if err != nil {
+		return err
+	}
+
+	netmonLog.SetLevel(level)
+
+	netmonLog.Formatter = &logrus.TextFormatter{TimestampFormat: time.RFC3339Nano}
+
+	hook, err := lSyslog.NewSyslogHook("", "", syslog.LOG_INFO|syslog.LOG_USER, netmonName)
+	if err != nil {
+		return err
+	}
+
+	netmonLog.AddHook(hook)
+
+	announceFields := logrus.Fields{
+		"runtime-path": n.runtimePath,
+		"debug":        n.debug,
+		"log-level":    n.logLevel,
+	}
+
+	n.logger().WithFields(announceFields).Info("announce")
+
+	return nil
+}
+
+func (n *netmon) listenNetlinkEvents() error {
+	if err := netlink.LinkSubscribe(n.linkUpdateCh, n.linkDoneCh); err != nil {
+		return err
+	}
+
+	return netlink.RouteSubscribe(n.rtUpdateCh, n.rtDoneCh)
+}
+
+// convertInterface converts a link and its IP addresses as defined by netlink
+// package, into the Interface structure format expected by kata-runtime to
+// describe an interface and its associated IP addresses.
+func convertInterface(linkAttrs *netlink.LinkAttrs, linkType string, addrs []netlink.Addr) pbTypes.Interface {
+	if linkAttrs == nil {
+		netmonLog.Warn("Link attributes are nil")
+		return pbTypes.Interface{}
+	}
+
+	var ipAddrs []*pbTypes.IPAddress
+
+	for _, addr := range addrs {
+		if addr.IPNet == nil {
+			continue
+		}
+
+		netMask, _ := addr.Mask.Size()
+
+		ipAddr := &pbTypes.IPAddress{
+			Address: addr.IP.String(),
+			Mask:    fmt.Sprintf("%d", netMask),
+		}
+
+		if addr.IP.To4() != nil {
+			ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V4)
+		} else {
+			ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V6)
+		}
+
+		ipAddrs = append(ipAddrs, ipAddr)
+	}
+
+	iface := pbTypes.Interface{
+		Device:      linkAttrs.Name,
+		Name:        linkAttrs.Name,
+		IPAddresses: ipAddrs,
+		Mtu:         uint64(linkAttrs.MTU),
+		HwAddr:      linkAttrs.HardwareAddr.String(),
+		Type:        linkType,
+	}
+
+	netmonLog.WithField("interface", iface).Debug("Interface converted")
+
+	return iface
+}
+
+// convertRoutes converts a list of routes as defined by netlink package,
+// into a list of Route structure format expected by kata-runtime to
+// describe a set of routes.
+func convertRoutes(netRoutes []netlink.Route) []pbTypes.Route {
+	var routes []pbTypes.Route
+
+	for _, netRoute := range netRoutes {
+		dst := ""
+
+		if netRoute.Protocol == unix.RTPROT_KERNEL {
+			continue
+		}
+
+		if netRoute.Dst != nil {
+			dst = netRoute.Dst.String()
+			if netRoute.Dst.IP.To4() != nil || netRoute.Dst.IP.To16() != nil {
+				dst = netRoute.Dst.String()
+			} else {
+				netmonLog.WithField("destination", netRoute.Dst.IP.String()).Warn("Unexpected network address format")
+			}
+		}
+
+		src := ""
+		if netRoute.Src != nil {
+			if netRoute.Src.To4() != nil || netRoute.Src.To16() != nil {
+				src = netRoute.Src.String()
+			} else {
+				netmonLog.WithField("source", netRoute.Src.String()).Warn("Unexpected network address format")
+			}
+		}
+
+		gw := ""
+		if netRoute.Gw != nil {
+			if netRoute.Gw.To4() != nil || netRoute.Gw.To16() != nil {
+				gw = netRoute.Gw.String()
+			} else {
+				netmonLog.WithField("gateway", netRoute.Gw.String()).Warn("Unexpected network address format")
+			}
+		}
+
+		dev := ""
+		iface, err := net.InterfaceByIndex(netRoute.LinkIndex)
+		if err == nil {
+			dev = iface.Name
+		}
+
+		route := pbTypes.Route{
+			Dest:    dst,
+			Gateway: gw,
+			Device:  dev,
+			Source:  src,
+			Scope:   uint32(netRoute.Scope),
+		}
+
+		routes = append(routes, route)
+	}
+
+	netmonLog.WithField("routes", routes).Debug("Routes converted")
+
+	return routes
+}
+
+// scanNetwork lists all the interfaces it can find inside the current
+// network namespace, and store them in-memory to keep track of them.
+func (n *netmon) scanNetwork() error {
+	links, err := n.netHandler.LinkList()
+	if err != nil {
+		return err
+	}
+
+	for _, link := range links {
+		addrs, err := n.netHandler.AddrList(link, netlinkFamily)
+		if err != nil {
+			return err
+		}
+
+		linkAttrs := link.Attrs()
+		if linkAttrs == nil {
+			continue
+		}
+
+		iface := convertInterface(linkAttrs, link.Type(), addrs)
+		n.netIfaces[linkAttrs.Index] = iface
+	}
+
+	n.logger().Debug("Network scanned")
+
+	return nil
+}
+
+func (n *netmon) storeDataToSend(data interface{}) error {
+	// Marshal the data structure into a JSON bytes array.
+	jsonArray, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+
+	// Store the JSON bytes array at the specified path.
+	return ioutil.WriteFile(n.sharedFile, jsonArray, storageFilePerm)
+}
+
+func (n *netmon) execKataCmd(subCmd string) error {
+	execCmd := exec.Command(n.runtimePath, kataCmd, subCmd, n.sandboxID, n.sharedFile)
+
+	n.logger().WithField("command", execCmd).Debug("Running runtime command")
+
+	// Make use of Run() to ensure the kata-runtime process has correctly
+	// terminated before to go further.
+	if err := execCmd.Run(); err != nil {
+		return err
+	}
+
+	// Remove the shared file after the command returned. At this point
+	// we know the content of the file is not going to be used anymore,
+	// and the file path can be reused for further commands.
+	return os.Remove(n.sharedFile)
+}
+
+func (n *netmon) addInterfaceCLI(iface pbTypes.Interface) error {
+	if err := n.storeDataToSend(iface); err != nil {
+		return err
+	}
+
+	return n.execKataCmd(kataCLIAddIfaceCmd)
+}
+
+func (n *netmon) delInterfaceCLI(iface pbTypes.Interface) error {
+	if err := n.storeDataToSend(iface); err != nil {
+		return err
+	}
+
+	return n.execKataCmd(kataCLIDelIfaceCmd)
+}
+
+func (n *netmon) updateRoutesCLI(routes []pbTypes.Route) error {
+	if err := n.storeDataToSend(routes); err != nil {
+		return err
+	}
+
+	return n.execKataCmd(kataCLIUpdtRoutesCmd)
+}
+
+func (n *netmon) updateRoutes() error {
+	// Get all the routes.
+	netlinkRoutes, err := n.netHandler.RouteList(nil, netlinkFamily)
+	if err != nil {
+		return err
+	}
+
+	// Translate them into Route structures.
+	routes := convertRoutes(netlinkRoutes)
+
+	// Update the routes through the Kata CLI.
+	return n.updateRoutesCLI(routes)
+}
+
+func (n *netmon) handleRTMNewAddr(ev netlink.LinkUpdate) error {
+	n.logger().Debug("Interface update not supported")
+	return nil
+}
+
+func (n *netmon) handleRTMDelAddr(ev netlink.LinkUpdate) error {
+	n.logger().Debug("Interface update not supported")
+	return nil
+}
+
+func (n *netmon) handleRTMNewLink(ev netlink.LinkUpdate) error {
+	// NEWLINK might be a lot of different things. We're interested in
+	// adding the interface (both to our list and by calling into the
+	// Kata CLI API) only if this has the flags UP and RUNNING, meaning
+	// we don't expect any further change on the interface, and that we
+	// are ready to add it.
+
+	linkAttrs := ev.Link.Attrs()
+	if linkAttrs == nil {
+		n.logger().Warn("The link attributes are nil")
+		return nil
+	}
+
+	// First, ignore if the interface name contains "kata". This way we
+	// are preventing from adding interfaces created by Kata Containers.
+	if strings.HasSuffix(linkAttrs.Name, kataSuffix) {
+		n.logger().Debugf("Ignore the interface %s because found %q",
+			linkAttrs.Name, kataSuffix)
+		return nil
+	}
+
+	// Check if the interface exist in the internal list.
+	if _, exist := n.netIfaces[int(ev.Index)]; exist {
+		n.logger().Debugf("Ignoring interface %s because already exist",
+			linkAttrs.Name)
+		return nil
+	}
+
+	// Now, check if the interface has been enabled to UP and RUNNING.
+	if (ev.Flags&unix.IFF_UP) != unix.IFF_UP ||
+		(ev.Flags&unix.IFF_RUNNING) != unix.IFF_RUNNING {
+		n.logger().Debugf("Ignore the interface %s because not UP and RUNNING",
+			linkAttrs.Name)
+		return nil
+	}
+
+	// Get the list of IP addresses associated with this interface.
+	addrs, err := n.netHandler.AddrList(ev.Link, netlinkFamily)
+	if err != nil {
+		return err
+	}
+
+	// Convert the interfaces in the appropriate structure format.
+	iface := convertInterface(linkAttrs, ev.Link.Type(), addrs)
+
+	// Add the interface through the Kata CLI.
+	if err := n.addInterfaceCLI(iface); err != nil {
+		return err
+	}
+
+	// Add the interface to the internal list.
+	n.netIfaces[linkAttrs.Index] = iface
+
+	// Complete by updating the routes.
+	return n.updateRoutes()
+}
+
+func (n *netmon) handleRTMDelLink(ev netlink.LinkUpdate) error {
+	// It can only delete if identical interface is found in the internal
+	// list of interfaces. Otherwise, the deletion will be ignored.
+	linkAttrs := ev.Link.Attrs()
+	if linkAttrs == nil {
+		n.logger().Warn("Link attributes are nil")
+		return nil
+	}
+
+	// First, ignore if the interface name contains "kata". This way we
+	// are preventing from deleting interfaces created by Kata Containers.
+	if strings.Contains(linkAttrs.Name, kataSuffix) {
+		n.logger().Debugf("Ignore the interface %s because found %q",
+			linkAttrs.Name, kataSuffix)
+		return nil
+	}
+
+	// Check if the interface exist in the internal list.
+	iface, exist := n.netIfaces[int(ev.Index)]
+	if !exist {
+		n.logger().Debugf("Ignoring interface %s because not found",
+			linkAttrs.Name)
+		return nil
+	}
+
+	if err := n.delInterfaceCLI(iface); err != nil {
+		return err
+	}
+
+	// Delete the interface from the internal list.
+	delete(n.netIfaces, linkAttrs.Index)
+
+	// Complete by updating the routes.
+	return n.updateRoutes()
+}
+
+func (n *netmon) handleRTMNewRoute(ev netlink.RouteUpdate) error {
+	// Add the route through updateRoutes(), only if the route refer to an
+	// interface that already exists in the internal list of interfaces.
+	if _, exist := n.netIfaces[ev.Route.LinkIndex]; !exist {
+		n.logger().Debugf("Ignoring route %+v since interface %d not found",
+			ev.Route, ev.Route.LinkIndex)
+		return nil
+	}
+
+	return n.updateRoutes()
+}
+
+func (n *netmon) handleRTMDelRoute(ev netlink.RouteUpdate) error {
+	// Remove the route through updateRoutes(), only if the route refer to
+	// an interface that already exists in the internal list of interfaces.
+	return n.updateRoutes()
+}
+
+func (n *netmon) handleLinkEvent(ev netlink.LinkUpdate) error {
+	n.logger().Debug("handleLinkEvent: netlink event received")
+
+	switch ev.Header.Type {
+	case unix.NLMSG_DONE:
+		n.logger().Debug("NLMSG_DONE")
+		return nil
+	case unix.NLMSG_ERROR:
+		n.logger().Error("NLMSG_ERROR")
+		return fmt.Errorf("Error while listening on netlink socket")
+	case unix.RTM_NEWADDR:
+		n.logger().Debug("RTM_NEWADDR")
+		return n.handleRTMNewAddr(ev)
+	case unix.RTM_DELADDR:
+		n.logger().Debug("RTM_DELADDR")
+		return n.handleRTMDelAddr(ev)
+	case unix.RTM_NEWLINK:
+		n.logger().Debug("RTM_NEWLINK")
+		return n.handleRTMNewLink(ev)
+	case unix.RTM_DELLINK:
+		n.logger().Debug("RTM_DELLINK")
+		return n.handleRTMDelLink(ev)
+	default:
+		n.logger().Warnf("Unknown msg type %v", ev.Header.Type)
+	}
+
+	return nil
+}
+
+func (n *netmon) handleRouteEvent(ev netlink.RouteUpdate) error {
+	n.logger().Debug("handleRouteEvent: netlink event received")
+
+	switch ev.Type {
+	case unix.RTM_NEWROUTE:
+		n.logger().Debug("RTM_NEWROUTE")
+		return n.handleRTMNewRoute(ev)
+	case unix.RTM_DELROUTE:
+		n.logger().Debug("RTM_DELROUTE")
+		return n.handleRTMDelRoute(ev)
+	default:
+		n.logger().Warnf("Unknown msg type %v", ev.Type)
+	}
+
+	return nil
+}
+
+func (n *netmon) handleEvents() (err error) {
+	for {
+		select {
+		case ev := <-n.linkUpdateCh:
+			if err = n.handleLinkEvent(ev); err != nil {
+				return err
+			}
+		case ev := <-n.rtUpdateCh:
+			if err = n.handleRouteEvent(ev); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func main() {
+	// Parse parameters.
+	params := parseOptions()
+
+	// Create netmon handler.
+	n, err := newNetmon(params)
+	if err != nil {
+		netmonLog.WithError(err).Fatal("newNetmon()")
+		os.Exit(1)
+	}
+	defer n.cleanup()
+
+	// Init logger.
+	if err := n.setupLogger(); err != nil {
+		netmonLog.WithError(err).Fatal("setupLogger()")
+		os.Exit(1)
+	}
+
+	// Setup signal handlers
+	n.setupSignalHandler()
+
+	// Scan the current interfaces.
+	if err := n.scanNetwork(); err != nil {
+		n.logger().WithError(err).Fatal("scanNetwork()")
+		os.Exit(1)
+	}
+
+	// Subscribe to the link listener.
+	if err := n.listenNetlinkEvents(); err != nil {
+		n.logger().WithError(err).Fatal("listenNetlinkEvents()")
+		os.Exit(1)
+	}
+
+	// Go into the main loop.
+	if err := n.handleEvents(); err != nil {
+		n.logger().WithError(err).Fatal("handleEvents()")
+		os.Exit(1)
+	}
+}

src/runtime/cmd/netmon/netmon_test.go

@@ -0,0 +1,701 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"testing"
+
+	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
+	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
+
+	"github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/assert"
+	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netns"
+	"golang.org/x/sys/unix"
+)
+
+const (
+	testSandboxID          = "123456789"
+	testRuntimePath        = "/foo/bar/test-runtime"
+	testLogLevel           = "info"
+	testStorageParentPath  = "/tmp/netmon"
+	testSharedFile         = "foo-shared.json"
+	testWrongNetlinkFamily = -1
+	testIfaceName          = "test_eth0"
+	testMTU                = 12345
+	testHwAddr             = "02:00:ca:fe:00:48"
+	testIPAddress          = "192.168.0.15"
+	testIPAddressWithMask  = "192.168.0.15/32"
+	testIP6Address         = "2001:db8:1::242:ac11:2"
+	testIP6AddressWithMask = "2001:db8:1::/64"
+	testScope              = 1
+	testTxQLen             = -1
+	testIfaceIndex         = 5
+)
+
+func skipUnlessRoot(t *testing.T) {
+	tc := ktu.NewTestConstraint(false)
+
+	if tc.NotValid(ktu.NeedRoot()) {
+		t.Skip("Test disabled as requires root user")
+	}
+}
+
+func TestNewNetmon(t *testing.T) {
+	skipUnlessRoot(t)
+
+	// Override storageParentPath
+	savedStorageParentPath := storageParentPath
+	storageParentPath = testStorageParentPath
+	defer func() {
+		storageParentPath = savedStorageParentPath
+	}()
+
+	params := netmonParams{
+		sandboxID:   testSandboxID,
+		runtimePath: testRuntimePath,
+		debug:       true,
+		logLevel:    testLogLevel,
+	}
+
+	expected := &netmon{
+		netmonParams: params,
+		storagePath:  filepath.Join(storageParentPath, params.sandboxID),
+		sharedFile:   filepath.Join(storageParentPath, params.sandboxID, sharedFile),
+	}
+
+	os.RemoveAll(expected.storagePath)
+
+	got, err := newNetmon(params)
+	assert.Nil(t, err)
+	assert.True(t, reflect.DeepEqual(expected.netmonParams, got.netmonParams),
+		"Got %+v\nExpected %+v", got.netmonParams, expected.netmonParams)
+	assert.True(t, reflect.DeepEqual(expected.storagePath, got.storagePath),
+		"Got %+v\nExpected %+v", got.storagePath, expected.storagePath)
+	assert.True(t, reflect.DeepEqual(expected.sharedFile, got.sharedFile),
+		"Got %+v\nExpected %+v", got.sharedFile, expected.sharedFile)
+
+	_, err = os.Stat(got.storagePath)
+	assert.Nil(t, err)
+
+	os.RemoveAll(got.storagePath)
+}
+
+func TestNewNetmonErrorWrongFamilyType(t *testing.T) {
+	// Override netlinkFamily
+	savedNetlinkFamily := netlinkFamily
+	netlinkFamily = testWrongNetlinkFamily
+	defer func() {
+		netlinkFamily = savedNetlinkFamily
+	}()
+
+	n, err := newNetmon(netmonParams{})
+	assert.NotNil(t, err)
+	assert.Nil(t, n)
+}
+
+func TestCleanup(t *testing.T) {
+	skipUnlessRoot(t)
+
+	// Override storageParentPath
+	savedStorageParentPath := storageParentPath
+	storageParentPath = testStorageParentPath
+	defer func() {
+		storageParentPath = savedStorageParentPath
+	}()
+
+	handler, err := netlink.NewHandle(netlinkFamily)
+	assert.Nil(t, err)
+
+	n := &netmon{
+		storagePath: filepath.Join(storageParentPath, testSandboxID),
+		linkDoneCh:  make(chan struct{}),
+		rtDoneCh:    make(chan struct{}),
+		netHandler:  handler,
+	}
+
+	err = os.MkdirAll(n.storagePath, storageDirPerm)
+	assert.Nil(t, err)
+	_, err = os.Stat(n.storagePath)
+	assert.Nil(t, err)
+
+	n.cleanup()
+
+	_, err = os.Stat(n.storagePath)
+	assert.NotNil(t, err)
+	_, ok := (<-n.linkDoneCh)
+	assert.False(t, ok)
+	_, ok = (<-n.rtDoneCh)
+	assert.False(t, ok)
+}
+
+func TestLogger(t *testing.T) {
+	fields := logrus.Fields{
+		"name":    netmonName,
+		"pid":     os.Getpid(),
+		"source":  "netmon",
+		"sandbox": testSandboxID,
+	}
+
+	expected := netmonLog.WithFields(fields)
+
+	n := &netmon{
+		netmonParams: netmonParams{
+			sandboxID: testSandboxID,
+		},
+	}
+
+	got := n.logger()
+	assert.True(t, reflect.DeepEqual(*expected, *got),
+		"Got %+v\nExpected %+v", *got, *expected)
+}
+
+func TestConvertInterface(t *testing.T) {
+	hwAddr, err := net.ParseMAC(testHwAddr)
+	assert.Nil(t, err)
+
+	addrs := []netlink.Addr{
+		{
+			IPNet: &net.IPNet{
+				IP: net.ParseIP(testIPAddress),
+			},
+		},
+		{
+			IPNet: &net.IPNet{
+				IP: net.ParseIP(testIP6Address),
+			},
+		},
+	}
+
+	linkAttrs := &netlink.LinkAttrs{
+		Name:         testIfaceName,
+		MTU:          testMTU,
+		HardwareAddr: hwAddr,
+	}
+
+	linkType := "link_type_test"
+
+	expected := pbTypes.Interface{
+		Device: testIfaceName,
+		Name:   testIfaceName,
+		Mtu:    uint64(testMTU),
+		HwAddr: testHwAddr,
+		IPAddresses: []*pbTypes.IPAddress{
+			{
+				Family:  utils.ConvertNetlinkFamily(netlink.FAMILY_V4),
+				Address: testIPAddress,
+				Mask:    "0",
+			},
+			{
+				Family:  utils.ConvertNetlinkFamily(netlink.FAMILY_V6),
+				Address: testIP6Address,
+				Mask:    "0",
+			},
+		},
+		Type: linkType,
+	}
+
+	got := convertInterface(linkAttrs, linkType, addrs)
+
+	assert.True(t, reflect.DeepEqual(expected, got),
+		"Got %+v\nExpected %+v", got, expected)
+}
+
+func TestConvertRoutes(t *testing.T) {
+	ip, ipNet, err := net.ParseCIDR(testIPAddressWithMask)
+	assert.Nil(t, err)
+	assert.NotNil(t, ipNet)
+
+	_, ip6Net, err := net.ParseCIDR(testIP6AddressWithMask)
+	assert.Nil(t, err)
+	assert.NotNil(t, ipNet)
+
+	routes := []netlink.Route{
+		{
+			Dst:       ipNet,
+			Src:       ip,
+			Gw:        ip,
+			LinkIndex: -1,
+			Scope:     testScope,
+		},
+		{
+			Dst:       ip6Net,
+			Src:       nil,
+			Gw:        nil,
+			LinkIndex: -1,
+			Scope:     testScope,
+		},
+	}
+
+	expected := []pbTypes.Route{
+		{
+			Dest:    testIPAddressWithMask,
+			Gateway: testIPAddress,
+			Source:  testIPAddress,
+			Scope:   uint32(testScope),
+		},
+		{
+			Dest:    testIP6AddressWithMask,
+			Gateway: "",
+			Source:  "",
+			Scope:   uint32(testScope),
+		},
+	}
+
+	got := convertRoutes(routes)
+	assert.True(t, reflect.DeepEqual(expected, got),
+		"Got %+v\nExpected %+v", got, expected)
+}
+
+type testTeardownNetwork func()
+
+func testSetupNetwork(t *testing.T) testTeardownNetwork {
+	skipUnlessRoot(t)
+
+	// new temporary namespace so we don't pollute the host
+	// lock thread since the namespace is thread local
+	runtime.LockOSThread()
+	var err error
+	ns, err := netns.New()
+	if err != nil {
+		t.Fatal("Failed to create newns", ns)
+	}
+
+	return func() {
+		ns.Close()
+		runtime.UnlockOSThread()
+	}
+}
+
+func testCreateDummyNetwork(t *testing.T, handler *netlink.Handle) (int, pbTypes.Interface) {
+	hwAddr, err := net.ParseMAC(testHwAddr)
+	assert.Nil(t, err)
+
+	link := &netlink.Dummy{
+		LinkAttrs: netlink.LinkAttrs{
+			MTU:          testMTU,
+			TxQLen:       testTxQLen,
+			Name:         testIfaceName,
+			HardwareAddr: hwAddr,
+		},
+	}
+
+	err = handler.LinkAdd(link)
+	assert.Nil(t, err)
+	err = handler.LinkSetUp(link)
+	assert.Nil(t, err)
+
+	attrs := link.Attrs()
+	assert.NotNil(t, attrs)
+
+	addrs, err := handler.AddrList(link, netlinkFamily)
+	assert.Nil(t, err)
+
+	var ipAddrs []*pbTypes.IPAddress
+
+	// Scan addresses for ipv6 link local address which is automatically assigned
+	for _, addr := range addrs {
+		if addr.IPNet == nil {
+			continue
+		}
+
+		netMask, _ := addr.Mask.Size()
+
+		ipAddr := &pbTypes.IPAddress{
+			Address: addr.IP.String(),
+			Mask:    fmt.Sprintf("%d", netMask),
+		}
+
+		if addr.IP.To4() != nil {
+			ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V4)
+		} else {
+			ipAddr.Family = utils.ConvertNetlinkFamily(netlink.FAMILY_V6)
+		}
+
+		ipAddrs = append(ipAddrs, ipAddr)
+	}
+
+	iface := pbTypes.Interface{
+		Device:      testIfaceName,
+		Name:        testIfaceName,
+		Mtu:         uint64(testMTU),
+		HwAddr:      testHwAddr,
+		Type:        link.Type(),
+		IPAddresses: ipAddrs,
+	}
+
+	return attrs.Index, iface
+}
+
+func TestScanNetwork(t *testing.T) {
+	tearDownNetworkCb := testSetupNetwork(t)
+	defer tearDownNetworkCb()
+
+	handler, err := netlink.NewHandle(netlinkFamily)
+	assert.Nil(t, err)
+	assert.NotNil(t, handler)
+	defer handler.Close()
+
+	idx, expected := testCreateDummyNetwork(t, handler)
+
+	n := &netmon{
+		netIfaces:  make(map[int]pbTypes.Interface),
+		netHandler: handler,
+	}
+
+	err = n.scanNetwork()
+	assert.Nil(t, err)
+	assert.True(t, reflect.DeepEqual(expected, n.netIfaces[idx]),
+		"Got %+v\nExpected %+v", n.netIfaces[idx], expected)
+}
+
+func TestStoreDataToSend(t *testing.T) {
+	var got pbTypes.Interface
+
+	expected := pbTypes.Interface{
+		Device: testIfaceName,
+		Name:   testIfaceName,
+		Mtu:    uint64(testMTU),
+		HwAddr: testHwAddr,
+	}
+
+	n := &netmon{
+		sharedFile: filepath.Join(testStorageParentPath, testSharedFile),
+	}
+
+	err := os.MkdirAll(testStorageParentPath, storageDirPerm)
+	defer os.RemoveAll(testStorageParentPath)
+	assert.Nil(t, err)
+
+	err = n.storeDataToSend(expected)
+	assert.Nil(t, err)
+
+	// Check the file has been created, check the content, and delete it.
+	_, err = os.Stat(n.sharedFile)
+	assert.Nil(t, err)
+	byteArray, err := ioutil.ReadFile(n.sharedFile)
+	assert.Nil(t, err)
+	err = json.Unmarshal(byteArray, &got)
+	assert.Nil(t, err)
+	assert.True(t, reflect.DeepEqual(expected, got),
+		"Got %+v\nExpected %+v", got, expected)
+}
+
+func TestExecKataCmdSuccess(t *testing.T) {
+	trueBinPath, err := exec.LookPath("true")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, trueBinPath)
+
+	params := netmonParams{
+		runtimePath: trueBinPath,
+	}
+
+	n := &netmon{
+		netmonParams: params,
+		sharedFile:   filepath.Join(testStorageParentPath, testSharedFile),
+	}
+
+	err = os.MkdirAll(testStorageParentPath, storageDirPerm)
+	assert.Nil(t, err)
+	defer os.RemoveAll(testStorageParentPath)
+
+	file, err := os.Create(n.sharedFile)
+	assert.Nil(t, err)
+	assert.NotNil(t, file)
+	file.Close()
+
+	_, err = os.Stat(n.sharedFile)
+	assert.Nil(t, err)
+
+	err = n.execKataCmd("")
+	assert.Nil(t, err)
+	_, err = os.Stat(n.sharedFile)
+	assert.NotNil(t, err)
+}
+
+func TestExecKataCmdFailure(t *testing.T) {
+	falseBinPath, err := exec.LookPath("false")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, falseBinPath)
+
+	params := netmonParams{
+		runtimePath: falseBinPath,
+	}
+
+	n := &netmon{
+		netmonParams: params,
+	}
+
+	err = n.execKataCmd("")
+	assert.NotNil(t, err)
+}
+
+func TestActionsCLI(t *testing.T) {
+	trueBinPath, err := exec.LookPath("true")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, trueBinPath)
+
+	params := netmonParams{
+		runtimePath: trueBinPath,
+	}
+
+	n := &netmon{
+		netmonParams: params,
+		sharedFile:   filepath.Join(testStorageParentPath, testSharedFile),
+	}
+
+	err = os.MkdirAll(testStorageParentPath, storageDirPerm)
+	assert.Nil(t, err)
+	defer os.RemoveAll(testStorageParentPath)
+
+	// Test addInterfaceCLI
+	err = n.addInterfaceCLI(pbTypes.Interface{})
+	assert.Nil(t, err)
+
+	// Test delInterfaceCLI
+	err = n.delInterfaceCLI(pbTypes.Interface{})
+	assert.Nil(t, err)
+
+	// Test updateRoutesCLI
+	err = n.updateRoutesCLI([]pbTypes.Route{})
+	assert.Nil(t, err)
+
+	tearDownNetworkCb := testSetupNetwork(t)
+	defer tearDownNetworkCb()
+
+	handler, err := netlink.NewHandle(netlinkFamily)
+	assert.Nil(t, err)
+	assert.NotNil(t, handler)
+	defer handler.Close()
+
+	n.netHandler = handler
+
+	// Test updateRoutes
+	err = n.updateRoutes()
+	assert.Nil(t, err)
+
+	// Test handleRTMDelRoute
+	err = n.handleRTMDelRoute(netlink.RouteUpdate{})
+	assert.Nil(t, err)
+}
+
+func TestHandleRTMNewAddr(t *testing.T) {
+	n := &netmon{}
+
+	err := n.handleRTMNewAddr(netlink.LinkUpdate{})
+	assert.Nil(t, err)
+}
+
+func TestHandleRTMDelAddr(t *testing.T) {
+	n := &netmon{}
+
+	err := n.handleRTMDelAddr(netlink.LinkUpdate{})
+	assert.Nil(t, err)
+}
+
+func TestHandleRTMNewLink(t *testing.T) {
+	n := &netmon{}
+	ev := netlink.LinkUpdate{
+		Link: &netlink.Dummy{},
+	}
+
+	// LinkAttrs is nil
+	err := n.handleRTMNewLink(ev)
+	assert.Nil(t, err)
+
+	// Link name contains "kata" suffix
+	ev = netlink.LinkUpdate{
+		Link: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: "foo_kata",
+			},
+		},
+	}
+	err = n.handleRTMNewLink(ev)
+	assert.Nil(t, err)
+
+	// Interface already exist in list
+	n.netIfaces = make(map[int]pbTypes.Interface)
+	n.netIfaces[testIfaceIndex] = pbTypes.Interface{}
+	ev = netlink.LinkUpdate{
+		Link: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: "foo0",
+			},
+		},
+	}
+	ev.Index = testIfaceIndex
+	err = n.handleRTMNewLink(ev)
+	assert.Nil(t, err)
+
+	// Flags are not up and running
+	n.netIfaces = make(map[int]pbTypes.Interface)
+	ev = netlink.LinkUpdate{
+		Link: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: "foo0",
+			},
+		},
+	}
+	ev.Index = testIfaceIndex
+	err = n.handleRTMNewLink(ev)
+	assert.Nil(t, err)
+
+	// Invalid link
+	n.netIfaces = make(map[int]pbTypes.Interface)
+	ev = netlink.LinkUpdate{
+		Link: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: "foo0",
+			},
+		},
+	}
+	ev.Index = testIfaceIndex
+	ev.Flags = unix.IFF_UP | unix.IFF_RUNNING
+	handler, err := netlink.NewHandle(netlinkFamily)
+	assert.Nil(t, err)
+	assert.NotNil(t, handler)
+	defer handler.Close()
+	n.netHandler = handler
+	err = n.handleRTMNewLink(ev)
+	assert.NotNil(t, err)
+}
+
+func TestHandleRTMDelLink(t *testing.T) {
+	n := &netmon{}
+	ev := netlink.LinkUpdate{
+		Link: &netlink.Dummy{},
+	}
+
+	// LinkAttrs is nil
+	err := n.handleRTMDelLink(ev)
+	assert.Nil(t, err)
+
+	// Link name contains "kata" suffix
+	ev = netlink.LinkUpdate{
+		Link: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: "foo_kata",
+			},
+		},
+	}
+	err = n.handleRTMDelLink(ev)
+	assert.Nil(t, err)
+
+	// Interface does not exist in list
+	n.netIfaces = make(map[int]pbTypes.Interface)
+	ev = netlink.LinkUpdate{
+		Link: &netlink.Dummy{
+			LinkAttrs: netlink.LinkAttrs{
+				Name: "foo0",
+			},
+		},
+	}
+	ev.Index = testIfaceIndex
+	err = n.handleRTMDelLink(ev)
+	assert.Nil(t, err)
+}
+
+func TestHandleRTMNewRouteIfaceNotFound(t *testing.T) {
+	n := &netmon{
+		netIfaces: make(map[int]pbTypes.Interface),
+	}
+
+	err := n.handleRTMNewRoute(netlink.RouteUpdate{})
+	assert.Nil(t, err)
+}
+
+func TestHandleLinkEvent(t *testing.T) {
+	n := &netmon{}
+	ev := netlink.LinkUpdate{}
+
+	// Unknown event
+	err := n.handleLinkEvent(ev)
+	assert.Nil(t, err)
+
+	// DONE event
+	ev.Header.Type = unix.NLMSG_DONE
+	err = n.handleLinkEvent(ev)
+	assert.Nil(t, err)
+
+	// ERROR event
+	ev.Header.Type = unix.NLMSG_ERROR
+	err = n.handleLinkEvent(ev)
+	assert.NotNil(t, err)
+
+	// NEWADDR event
+	ev.Header.Type = unix.RTM_NEWADDR
+	err = n.handleLinkEvent(ev)
+	assert.Nil(t, err)
+
+	// DELADDR event
+	ev.Header.Type = unix.RTM_DELADDR
+	err = n.handleLinkEvent(ev)
+	assert.Nil(t, err)
+
+	// NEWLINK event
+	ev.Header.Type = unix.RTM_NEWLINK
+	ev.Link = &netlink.Dummy{}
+	err = n.handleLinkEvent(ev)
+	assert.Nil(t, err)
+
+	// DELLINK event
+	ev.Header.Type = unix.RTM_DELLINK
+	ev.Link = &netlink.Dummy{}
+	err = n.handleLinkEvent(ev)
+	assert.Nil(t, err)
+}
+
+func TestHandleRouteEvent(t *testing.T) {
+	n := &netmon{}
+	ev := netlink.RouteUpdate{}
+
+	// Unknown event
+	err := n.handleRouteEvent(ev)
+	assert.Nil(t, err)
+
+	// RTM_NEWROUTE event
+	ev.Type = unix.RTM_NEWROUTE
+	err = n.handleRouteEvent(ev)
+	assert.Nil(t, err)
+
+	trueBinPath, err := exec.LookPath("true")
+	assert.Nil(t, err)
+	assert.NotEmpty(t, trueBinPath)
+
+	n.runtimePath = trueBinPath
+	n.sharedFile = filepath.Join(testStorageParentPath, testSharedFile)
+
+	err = os.MkdirAll(testStorageParentPath, storageDirPerm)
+	assert.Nil(t, err)
+	defer os.RemoveAll(testStorageParentPath)
+
+	tearDownNetworkCb := testSetupNetwork(t)
+	defer tearDownNetworkCb()
+
+	handler, err := netlink.NewHandle(netlinkFamily)
+	assert.Nil(t, err)
+	assert.NotNil(t, handler)
+	defer handler.Close()
+
+	n.netHandler = handler
+
+	// RTM_DELROUTE event
+	ev.Type = unix.RTM_DELROUTE
+	err = n.handleRouteEvent(ev)
+	assert.Nil(t, err)
+}

src/runtime/config/configuration-acrn.toml.in

@@ -147,6 +147,21 @@ block_device_driver = "@DEFBLOCKSTORAGEDRIVER_ACRN@"
 # (default: 30)
 #dial_timeout = 30
 
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
 # system log
@@ -202,6 +217,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
 # If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
 # This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
 # `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
 # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
 # (like OVS) directly.

src/runtime/config/configuration-clh.toml.in

@@ -169,6 +169,22 @@ block_device_driver = "virtio-blk"
 # (default: 30)
 #dial_timeout = 30
 
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
+
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
 # system log
@@ -224,6 +240,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
 # If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
 # This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
 # `disable_new_netns` conflicts with `internetworking_model=bridged` and `internetworking_model=macvtap`. It works only
 # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
 # (like OVS) directly.

src/runtime/config/configuration-fc.toml.in

@@ -282,6 +282,21 @@ kernel_modules=[]
 # (default: 30)
 #dial_timeout = 30
 
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
 # system log
@@ -330,6 +345,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
 # If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
 # This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
 # `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
 # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
 # (like OVS) directly.

src/runtime/config/configuration-qemu.toml.in

@@ -458,6 +458,21 @@ kernel_modules=[]
 # (default: 30)
 #dial_timeout = 30
 
+[netmon]
+# If enabled, the network monitoring process gets started when the
+# sandbox is created. This allows for the detection of some additional
+# network being added to the existing network namespace, after the
+# sandbox has been created.
+# (default: disabled)
+#enable_netmon = true
+
+# Specify the path to the netmon binary.
+path = "@NETMONPATH@"
+
+# If enabled, netmon messages will be sent to the system log
+# (default: disabled)
+#enable_debug = true
+
 [runtime]
 # If enabled, the runtime will log additional debug messages to the
 # system log
@@ -506,6 +521,7 @@ disable_guest_seccomp=@DEFDISABLEGUESTSECCOMP@
 
 # If enabled, the runtime will not create a network namespace for shim and hypervisor processes.
 # This option may have some potential impacts to your host. It should only be used when you know what you're doing.
+# `disable_new_netns` conflicts with `enable_netmon`
 # `disable_new_netns` conflicts with `internetworking_model=tcfilter` and `internetworking_model=macvtap`. It works only
 # with `internetworking_model=none`. The tap device will be in the host network namespace and can connect to a bridge
 # (like OVS) directly.

src/runtime/pkg/containerd-shim-v2/create.go

@@ -34,9 +34,9 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 )
 
 type startManagementServerFunc func(s *service, ctx context.Context, ociSpec *specs.Spec)

src/runtime/pkg/containerd-shim-v2/create_test.go

@@ -328,6 +328,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 	kernelParams := "foo=bar xyz"
 	imagePath := path.Join(dir, "image")
 	shimPath := path.Join(dir, "shim")
+	netmonPath := path.Join(dir, "netmon")
 	logDir := path.Join(dir, "logs")
 	logPath := path.Join(logDir, "runtime.log")
 	machineType := "machineType"
@@ -348,6 +349,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config string, err err
 		KernelParams:         kernelParams,
 		MachineType:          machineType,
 		ShimPath:             shimPath,
+		NetmonPath:           netmonPath,
 		LogPath:              logPath,
 		DisableBlock:         disableBlockDevice,
 		BlockDeviceDriver:    blockDeviceDriver,

src/runtime/pkg/containerd-shim-v2/errors.go

@@ -13,7 +13,7 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 )
 
 // toGRPC maps the virtcontainers error into a grpc error,

src/runtime/pkg/containerd-shim-v2/errors_test.go

@@ -10,7 +10,7 @@ import (
 	"syscall"
 	"testing"
 
-	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/stretchr/testify/assert"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"

src/runtime/pkg/containerd-shim-v2/service.go

@@ -31,9 +31,9 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )
 
@@ -953,12 +953,6 @@ func (s *service) Shutdown(ctx context.Context, r *taskAPI.ShutdownRequest) (_ *
 	// exited when shimv2 terminated. Thus here to do the last cleanup of the hypervisor.
 	syscall.Kill(int(s.hpid), syscall.SIGKILL)
 
-	// os.Exit() will terminate program immediately, the defer functions won't be executed,
-	// so we add defer functions again before os.Exit().
-	// Refer to https://pkg.go.dev/os#Exit
-	shimLog.WithField("container", r.ID).Debug("Shutdown() end")
-	rpcDurationsHistogram.WithLabelValues("shutdown").Observe(float64(time.Since(start).Nanoseconds() / int64(time.Millisecond)))
-
 	os.Exit(0)
 
 	// This will never be called, but this is only there to make sure the

src/runtime/pkg/containerd-shim-v2/utils.go

@@ -16,9 +16,9 @@ import (
 	"github.com/containerd/containerd/mount"
 	cdshim "github.com/containerd/containerd/runtime/v2/shim"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 )
 
 func cReap(s *service, status int, id, execid string, exitat time.Time) {

src/runtime/pkg/containerd-shim-v2/utils_test.go

@@ -16,8 +16,8 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 	"github.com/pkg/errors"
 )

src/runtime/pkg/containerd-shim-v2/wait.go

@@ -17,7 +17,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc/codes"
 
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 )
 
 const defaultCheckInterval = 1 * time.Second

src/runtime/pkg/katatestutils/utils.go

@@ -215,6 +215,7 @@ type RuntimeConfigOptions struct {
 	KernelParams         string
 	MachineType          string
 	ShimPath             string
+	NetmonPath           string
 	LogPath              string
 	BlockDeviceDriver    string
 	SharedFS             string
@@ -236,6 +237,7 @@ type RuntimeConfigOptions struct {
 	RuntimeDebug         bool
 	RuntimeTrace         bool
 	ShimDebug            bool
+	NetmonDebug          bool
 	AgentDebug           bool
 	AgentTrace           bool
 	EnablePprof          bool
@@ -329,6 +331,10 @@ func MakeRuntimeConfigFileData(config RuntimeConfigOptions) string {
 	enable_debug = ` + strconv.FormatBool(config.AgentDebug) + `
 	enable_tracing = ` + strconv.FormatBool(config.AgentTrace) + `
 
+	[netmon]
+	path = "` + config.NetmonPath + `"
+	enable_debug = ` + strconv.FormatBool(config.NetmonDebug) + `
+
 	[runtime]
 	enable_debug = ` + strconv.FormatBool(config.RuntimeDebug) + `
 	enable_tracing = ` + strconv.FormatBool(config.RuntimeTrace) + `

src/runtime/pkg/katautils/config-settings.go.in

@@ -97,3 +97,5 @@ const defaultVMCacheEndpoint string = "/var/run/kata-containers/cache.sock"
 
 // Default config file used by stateless systems.
 var defaultRuntimeConfiguration = "@CONFIG_PATH@"
+
+var defaultNetmonPath = "/usr/libexec/kata-containers/kata-netmon"

src/runtime/pkg/katautils/config.go

@@ -17,10 +17,10 @@ import (
 	"github.com/BurntSushi/toml"
 	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
 )
@@ -56,6 +56,7 @@ type tomlConfig struct {
 	Agent      map[string]agent
 	Runtime    runtime
 	Image      image
+	Netmon     netmon
 	Factory    factory
 }
 
@@ -161,6 +162,12 @@ type agent struct {
 	DialTimeout         uint32   `toml:"dial_timeout"`
 }
 
+type netmon struct {
+	Path   string `toml:"path"`
+	Debug  bool   `toml:"enable_debug"`
+	Enable bool   `toml:"enable_netmon"`
+}
+
 func (h hypervisor) path() (string, error) {
 	p := h.Path
 
@@ -499,6 +506,22 @@ func (a agent) kernelModules() []string {
 	return a.KernelModules
 }
 
+func (n netmon) enable() bool {
+	return n.Enable
+}
+
+func (n netmon) path() string {
+	if n.Path == "" {
+		return defaultNetmonPath
+	}
+
+	return n.Path
+}
+
+func (n netmon) debug() bool {
+	return n.Debug
+}
+
 func newFirecrackerHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 	hypervisor, err := h.path()
 	if err != nil {
@@ -991,6 +1014,12 @@ func updateRuntimeConfig(configPath string, tomlConf tomlConfig, config *oci.Run
 	}
 	config.FactoryConfig = fConfig
 
+	config.NetmonConfig = vc.NetmonConfig{
+		Path:   tomlConf.Netmon.path(),
+		Debug:  tomlConf.Netmon.debug(),
+		Enable: tomlConf.Netmon.enable(),
+	}
+
 	err = SetKernelParams(config)
 	if err != nil {
 		return err
@@ -1233,6 +1262,9 @@ func checkConfig(config oci.RuntimeConfig) error {
 // Because it is an expert option and conflicts with some other common configs.
 func checkNetNsConfig(config oci.RuntimeConfig) error {
 	if config.DisableNewNetNs {
+		if config.NetmonConfig.Enable {
+			return fmt.Errorf("config disable_new_netns conflicts with enable_netmon")
+		}
 		if config.InterNetworkModel != vc.NetXConnectNoneModel {
 			return fmt.Errorf("config disable_new_netns only works with 'none' internetworking_model")
 		}

src/runtime/pkg/katautils/config_test.go

@@ -19,8 +19,8 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/stretchr/testify/assert"
 )
@@ -29,6 +29,7 @@ var (
 	hypervisorDebug = false
 	runtimeDebug    = false
 	runtimeTrace    = false
+	netmonDebug     = false
 	agentDebug      = false
 	agentTrace      = false
 	enablePprof     = true
@@ -73,6 +74,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 	kernelPath := path.Join(dir, "kernel")
 	kernelParams := "foo=bar xyz"
 	imagePath := path.Join(dir, "image")
+	netmonPath := path.Join(dir, "netmon")
 	logDir := path.Join(dir, "logs")
 	logPath := path.Join(logDir, "runtime.log")
 	machineType := "machineType"
@@ -93,6 +95,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		ImagePath:            imagePath,
 		KernelParams:         kernelParams,
 		MachineType:          machineType,
+		NetmonPath:           netmonPath,
 		LogPath:              logPath,
 		DefaultGuestHookPath: defaultGuestHookPath,
 		DisableBlock:         disableBlockDevice,
@@ -108,6 +111,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		HypervisorDebug:      hypervisorDebug,
 		RuntimeDebug:         runtimeDebug,
 		RuntimeTrace:         runtimeTrace,
+		NetmonDebug:          netmonDebug,
 		AgentDebug:           agentDebug,
 		AgentTrace:           agentTrace,
 		SharedFS:             sharedFS,
@@ -176,6 +180,12 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 		LongLiveConn: true,
 	}
 
+	netmonConfig := vc.NetmonConfig{
+		Path:   netmonPath,
+		Debug:  false,
+		Enable: false,
+	}
+
 	factoryConfig := oci.FactoryConfig{
 		TemplatePath:    defaultTemplatePath,
 		VMCacheEndpoint: defaultVMCacheEndpoint,
@@ -187,6 +197,7 @@ func createAllRuntimeConfigFiles(dir, hypervisor string) (config testRuntimeConf
 
 		AgentConfig: agentConfig,
 
+		NetmonConfig:    netmonConfig,
 		DisableNewNetNs: disableNewNetNs,
 		EnablePprof:     enablePprof,
 		JaegerEndpoint:  jaegerEndpoint,
@@ -482,6 +493,7 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 	defaultHypervisorPath = hypervisorPath
 	jailerPath := path.Join(dir, "jailer")
 	defaultJailerPath = jailerPath
+	netmonPath := path.Join(dir, "netmon")
 
 	imagePath := path.Join(dir, "image.img")
 	initrdPath := path.Join(dir, "initrd.img")
@@ -523,6 +535,8 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 	[agent.kata]
 	debug_console_enabled=true
 	kernel_modules=["a", "b", "c"]
+	[netmon]
+	path = "` + netmonPath + `"
 `
 
 	orgVHostVSockDevicePath := utils.VHostVSockDevicePath
@@ -547,6 +561,11 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		t.Error(err)
 	}
 
+	err = createEmptyFile(netmonPath)
+	if err != nil {
+		t.Error(err)
+	}
+
 	_, config, err := LoadConfiguration(configPath, false)
 	if err != nil {
 		t.Fatal(err)
@@ -578,6 +597,12 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 		KernelModules:      []string{"a", "b", "c"},
 	}
 
+	expectedNetmonConfig := vc.NetmonConfig{
+		Path:   netmonPath,
+		Debug:  false,
+		Enable: false,
+	}
+
 	expectedFactoryConfig := oci.FactoryConfig{
 		TemplatePath:    defaultTemplatePath,
 		VMCacheEndpoint: defaultVMCacheEndpoint,
@@ -589,6 +614,8 @@ func TestMinimalRuntimeConfig(t *testing.T) {
 
 		AgentConfig: expectedAgentConfig,
 
+		NetmonConfig: expectedNetmonConfig,
+
 		FactoryConfig: expectedFactoryConfig,
 	}
 	err = SetKernelParams(&expectedConfig)
@@ -1526,6 +1553,9 @@ func TestCheckNetNsConfig(t *testing.T) {
 
 	config := oci.RuntimeConfig{
 		DisableNewNetNs: true,
+		NetmonConfig: vc.NetmonConfig{
+			Enable: true,
+		},
 	}
 	err := checkNetNsConfig(config)
 	assert.Error(err)

src/runtime/pkg/katautils/create.go

@@ -14,9 +14,9 @@ import (
 	"strings"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	vf "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/factory"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -120,9 +120,6 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec specs.Spec, runtimeCo
 		return nil, vc.Process{}, err
 	}
 
-	// setup shared path in hypervisor config:
-	sandboxConfig.HypervisorConfig.SharedPath = vc.GetSharePath(containerID)
-
 	if err := checkForFIPS(&sandboxConfig); err != nil {
 		return nil, vc.Process{}, err
 	}

src/runtime/pkg/katautils/create_test.go

@@ -19,9 +19,9 @@ import (
 	"testing"
 
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/vcmock"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/assert"

src/runtime/virtcontainers/README.md

@@ -17,7 +17,7 @@ or the [Kubernetes CRI][cri]) to the `virtcontainers` API.
 `virtcontainers` was used as a foundational package for the [Clear Containers][cc] [runtime][cc-runtime] implementation.
 
 [oci]: https://github.com/opencontainers/runtime-spec
-[cri]: https://git.k8s.io/community/contributors/devel/sig-node/container-runtime-interface.md
+[cri]: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-node/container-runtime-interface.md
 [cc]: https://github.com/clearcontainers/
 [cc-runtime]: https://github.com/clearcontainers/runtime/
 

src/runtime/virtcontainers/acrn.go

@@ -7,6 +7,7 @@ package virtcontainers
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"os/exec"
@@ -19,13 +20,12 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -39,13 +39,11 @@ var acrnTracingTags = map[string]string{
 
 // Since ACRN is using the store in a quite abnormal way, let's first draw it back from store to here
 
-/*
 // UUIDPathSuffix is the suffix used for uuid storage
 const (
 	UUIDPathSuffix = "uuid"
 	uuidFile       = "uuid.json"
 )
-*/
 
 // ACRN currently supports only known UUIDs for security
 // reasons (FuSa). When launching VM, only these pre-defined
@@ -314,7 +312,7 @@ func (a *Acrn) setup(ctx context.Context, id string, hypervisorConfig *Hyperviso
 
 		// The path might already exist, but in case of VM templating,
 		// we have to create it since the sandbox has not created it yet.
-		if err = os.MkdirAll(filepath.Join(a.config.RunStorePath, id), DirMode); err != nil {
+		if err = os.MkdirAll(filepath.Join(a.store.RunStoragePath(), id), DirMode); err != nil {
 			return err
 		}
 
@@ -440,7 +438,7 @@ func (a *Acrn) StartVM(ctx context.Context, timeoutSecs int) error {
 		a.Logger().WithField("default-kernel-parameters", formatted).Debug()
 	}
 
-	vmPath := filepath.Join(a.config.VMStorePath, a.id)
+	vmPath := filepath.Join(a.store.RunVMStoragePath(), a.id)
 	err := os.MkdirAll(vmPath, DirMode)
 	if err != nil {
 		return err
@@ -636,7 +634,7 @@ func (a *Acrn) GetVMConsole(ctx context.Context, id string) (string, string, err
 	span, _ := katatrace.Trace(ctx, a.Logger(), "GetVMConsole", acrnTracingTags, map[string]string{"sandbox_id": a.id})
 	defer span.End()
 
-	consoleURL, err := utils.BuildSocketPath(a.config.VMStorePath, id, acrnConsoleSocket)
+	consoleURL, err := utils.BuildSocketPath(a.store.RunVMStoragePath(), id, acrnConsoleSocket)
 	if err != nil {
 		return consoleProtoUnix, "", err
 	}
@@ -700,14 +698,14 @@ func (a *Acrn) toGrpc(ctx context.Context) ([]byte, error) {
 	return nil, errors.New("acrn is not supported by VM cache")
 }
 
-func (a *Acrn) Save() (s hv.HypervisorState) {
+func (a *Acrn) Save() (s persistapi.HypervisorState) {
 	s.Pid = a.state.PID
 	s.Type = string(AcrnHypervisor)
 	s.UUID = a.state.UUID
 	return
 }
 
-func (a *Acrn) Load(s hv.HypervisorState) {
+func (a *Acrn) Load(s persistapi.HypervisorState) {
 	a.state.PID = s.Pid
 	a.state.UUID = s.UUID
 }
@@ -721,7 +719,7 @@ func (a *Acrn) Check() error {
 }
 
 func (a *Acrn) GenerateSocket(id string) (interface{}, error) {
-	return generateVMSocket(id, a.config.VMStorePath)
+	return generateVMSocket(id, a.store.RunVMStoragePath())
 }
 
 // GetACRNUUIDBytes returns UUID bytes that is used for VM creation
@@ -784,36 +782,38 @@ func (a *Acrn) GetMaxSupportedACRNVM() (uint8, error) {
 }
 
 func (a *Acrn) storeInfo() error {
-	/*
-		relPath := filepath.Join(UUIDPathSuffix, uuidFile)
+	relPath := filepath.Join(UUIDPathSuffix, uuidFile)
 
-		jsonOut, err := json.Marshal(a.info)
-		if err != nil {
-			return fmt.Errorf("Could not marshal data: %s", err)
-		}
+	jsonOut, err := json.Marshal(a.info)
+	if err != nil {
+		return fmt.Errorf("Could not marshal data: %s", err)
+	}
 
-		if err := a.store.GlobalWrite(relPath, jsonOut); err != nil {
-			return fmt.Errorf("failed to write uuid to file: %v", err)
-		}*/
+	if err := a.store.GlobalWrite(relPath, jsonOut); err != nil {
+		return fmt.Errorf("failed to write uuid to file: %v", err)
+	}
 
 	return nil
 }
 
 func (a *Acrn) loadInfo() error {
-	/*
-		relPath := filepath.Join(UUIDPathSuffix, uuidFile)
-			data, err := a.store.GlobalRead(relPath)
-			if err != nil {
-				return fmt.Errorf("failed to read uuid from file: %v", err)
-			}
+	relPath := filepath.Join(UUIDPathSuffix, uuidFile)
 
-			if err := json.Unmarshal(data, &a.info); err != nil {
-				return fmt.Errorf("failed to unmarshal uuid info: %v", err)
-			}*/
+	data, err := a.store.GlobalRead(relPath)
+	if err != nil {
+		return fmt.Errorf("failed to read uuid from file: %v", err)
+	}
 
+	if err := json.Unmarshal(data, &a.info); err != nil {
+		return fmt.Errorf("failed to unmarshal uuid info: %v", err)
+	}
 	return nil
 }
 
 func (a *Acrn) IsRateLimiterBuiltin() bool {
 	return false
 }
+
+func (a *Acrn) setSandbox(sandbox *Sandbox) {
+	a.sandbox = sandbox
+}

src/runtime/virtcontainers/acrn_test.go

@@ -199,15 +199,11 @@ func TestAcrnGetSandboxConsole(t *testing.T) {
 	assert.NoError(err)
 
 	a := &Acrn{
-		ctx: context.Background(),
-		config: HypervisorConfig{
-			VMStorePath:  store.RunVMStoragePath(),
-			RunStorePath: store.RunStoragePath(),
-		},
+		ctx:   context.Background(),
 		store: store,
 	}
 	sandboxID := "testSandboxID"
-	expected := filepath.Join(store.RunVMStoragePath(), sandboxID, consoleSocket)
+	expected := filepath.Join(a.store.RunVMStoragePath(), sandboxID, consoleSocket)
 
 	proto, result, err := a.GetVMConsole(a.ctx, sandboxID)
 	assert.NoError(err)
@@ -223,10 +219,6 @@ func TestAcrnCreateVM(t *testing.T) {
 
 	a := &Acrn{
 		store: store,
-		config: HypervisorConfig{
-			VMStorePath:  store.RunVMStoragePath(),
-			RunStorePath: store.RunStoragePath(),
-		},
 	}
 
 	sandbox := &Sandbox{

src/runtime/virtcontainers/agent.go

@@ -12,8 +12,8 @@ import (
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/net/context"
 )

src/runtime/virtcontainers/api.go

@@ -14,7 +14,7 @@ import (
 	deviceConfig "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cgroups"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/sirupsen/logrus"
 )
 
@@ -35,7 +35,7 @@ var virtLog = logrus.WithField("source", "virtcontainers")
 func SetLogger(ctx context.Context, logger *logrus.Entry) {
 	fields := virtLog.Data
 	virtLog = logger.WithFields(fields)
-	SetHypervisorLogger(virtLog) // TODO: this will move to hypervisors pkg
+
 	deviceApi.SetLogger(virtLog)
 	compatoci.SetLogger(virtLog)
 	deviceConfig.SetLogger(virtLog)

src/runtime/virtcontainers/bridgedmacvlan_endpoint.go

@@ -11,13 +11,13 @@ import (
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 )
 
-var macvlanTrace = getNetworkTrace(MacvlanEndpointType)
+var macvlanTrace = getNetworkTrace(BridgedMacvlanEndpointType)
 
-// MacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
-type MacvlanEndpoint struct {
+// BridgedMacvlanEndpoint represents a macvlan endpoint that is bridged to the VM
+type BridgedMacvlanEndpoint struct {
 	EndpointType       EndpointType
 	PCIPath            vcTypes.PciPath
 	EndpointProperties NetworkInfo
@@ -26,9 +26,9 @@ type MacvlanEndpoint struct {
 	TxRateLimiter      bool
 }
 
-func createMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*MacvlanEndpoint, error) {
+func createBridgedMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetInterworkingModel) (*BridgedMacvlanEndpoint, error) {
 	if idx < 0 {
-		return &MacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
+		return &BridgedMacvlanEndpoint{}, fmt.Errorf("invalid network endpoint index: %d", idx)
 	}
 
 	netPair, err := createNetworkInterfacePair(idx, ifName, interworkingModel)
@@ -36,9 +36,9 @@ func createMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetI
 		return nil, err
 	}
 
-	endpoint := &MacvlanEndpoint{
+	endpoint := &BridgedMacvlanEndpoint{
 		NetPair:      netPair,
-		EndpointType: MacvlanEndpointType,
+		EndpointType: BridgedMacvlanEndpointType,
 	}
 	if ifName != "" {
 		endpoint.NetPair.VirtIface.Name = ifName
@@ -48,49 +48,49 @@ func createMacvlanNetworkEndpoint(idx int, ifName string, interworkingModel NetI
 }
 
 // Properties returns properties of the interface.
-func (endpoint *MacvlanEndpoint) Properties() NetworkInfo {
+func (endpoint *BridgedMacvlanEndpoint) Properties() NetworkInfo {
 	return endpoint.EndpointProperties
 }
 
 // Name returns name of the veth interface in the network pair.
-func (endpoint *MacvlanEndpoint) Name() string {
+func (endpoint *BridgedMacvlanEndpoint) Name() string {
 	return endpoint.NetPair.VirtIface.Name
 }
 
 // HardwareAddr returns the mac address that is assigned to the tap interface
 // in th network pair.
-func (endpoint *MacvlanEndpoint) HardwareAddr() string {
+func (endpoint *BridgedMacvlanEndpoint) HardwareAddr() string {
 	return endpoint.NetPair.TAPIface.HardAddr
 }
 
 // Type identifies the endpoint as a bridged macvlan endpoint.
-func (endpoint *MacvlanEndpoint) Type() EndpointType {
+func (endpoint *BridgedMacvlanEndpoint) Type() EndpointType {
 	return endpoint.EndpointType
 }
 
 // SetProperties sets the properties for the endpoint.
-func (endpoint *MacvlanEndpoint) SetProperties(properties NetworkInfo) {
+func (endpoint *BridgedMacvlanEndpoint) SetProperties(properties NetworkInfo) {
 	endpoint.EndpointProperties = properties
 }
 
 // PciPath returns the PCI path of the endpoint.
-func (endpoint *MacvlanEndpoint) PciPath() vcTypes.PciPath {
+func (endpoint *BridgedMacvlanEndpoint) PciPath() vcTypes.PciPath {
 	return endpoint.PCIPath
 }
 
 // SetPciPath sets the PCI path of the endpoint.
-func (endpoint *MacvlanEndpoint) SetPciPath(pciPath vcTypes.PciPath) {
+func (endpoint *BridgedMacvlanEndpoint) SetPciPath(pciPath vcTypes.PciPath) {
 	endpoint.PCIPath = pciPath
 }
 
 // NetworkPair returns the network pair of the endpoint.
-func (endpoint *MacvlanEndpoint) NetworkPair() *NetworkInterfacePair {
+func (endpoint *BridgedMacvlanEndpoint) NetworkPair() *NetworkInterfacePair {
 	return &endpoint.NetPair
 }
 
 // Attach for virtual endpoint bridges the network pair and adds the
 // tap interface of the network pair to the hypervisor.
-func (endpoint *MacvlanEndpoint) Attach(ctx context.Context, s *Sandbox) error {
+func (endpoint *BridgedMacvlanEndpoint) Attach(ctx context.Context, s *Sandbox) error {
 	span, ctx := macvlanTrace(ctx, "Attach", endpoint)
 	defer span.End()
 
@@ -105,7 +105,7 @@ func (endpoint *MacvlanEndpoint) Attach(ctx context.Context, s *Sandbox) error {
 
 // Detach for the virtual endpoint tears down the tap and bridge
 // created for the veth interface.
-func (endpoint *MacvlanEndpoint) Detach(ctx context.Context, netNsCreated bool, netNsPath string) error {
+func (endpoint *BridgedMacvlanEndpoint) Detach(ctx context.Context, netNsCreated bool, netNsPath string) error {
 	// The network namespace would have been deleted at this point
 	// if it has not been created by virtcontainers.
 	if !netNsCreated {
@@ -121,49 +121,49 @@ func (endpoint *MacvlanEndpoint) Detach(ctx context.Context, netNsCreated bool,
 }
 
 // HotAttach for bridged macvlan endpoint not supported yet
-func (endpoint *MacvlanEndpoint) HotAttach(ctx context.Context, h Hypervisor) error {
-	return fmt.Errorf("MacvlanEndpoint does not support Hot attach")
+func (endpoint *BridgedMacvlanEndpoint) HotAttach(ctx context.Context, h Hypervisor) error {
+	return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot attach")
 }
 
 // HotDetach for bridged macvlan endpoint not supported yet
-func (endpoint *MacvlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
-	return fmt.Errorf("MacvlanEndpoint does not support Hot detach")
+func (endpoint *BridgedMacvlanEndpoint) HotDetach(ctx context.Context, h Hypervisor, netNsCreated bool, netNsPath string) error {
+	return fmt.Errorf("BridgedMacvlanEndpoint does not support Hot detach")
 }
 
-func (endpoint *MacvlanEndpoint) save() persistapi.NetworkEndpoint {
+func (endpoint *BridgedMacvlanEndpoint) save() persistapi.NetworkEndpoint {
 	netpair := saveNetIfPair(&endpoint.NetPair)
 
 	return persistapi.NetworkEndpoint{
 		Type: string(endpoint.Type()),
-		Macvlan: &persistapi.MacvlanEndpoint{
+		BridgedMacvlan: &persistapi.BridgedMacvlanEndpoint{
 			NetPair: *netpair,
 		},
 	}
 }
 
-func (endpoint *MacvlanEndpoint) load(s persistapi.NetworkEndpoint) {
-	endpoint.EndpointType = MacvlanEndpointType
+func (endpoint *BridgedMacvlanEndpoint) load(s persistapi.NetworkEndpoint) {
+	endpoint.EndpointType = BridgedMacvlanEndpointType
 
-	if s.Macvlan != nil {
-		netpair := loadNetIfPair(&s.Macvlan.NetPair)
+	if s.BridgedMacvlan != nil {
+		netpair := loadNetIfPair(&s.BridgedMacvlan.NetPair)
 		endpoint.NetPair = *netpair
 	}
 }
 
-func (endpoint *MacvlanEndpoint) GetRxRateLimiter() bool {
+func (endpoint *BridgedMacvlanEndpoint) GetRxRateLimiter() bool {
 	return endpoint.RxRateLimiter
 }
 
-func (endpoint *MacvlanEndpoint) SetRxRateLimiter() error {
+func (endpoint *BridgedMacvlanEndpoint) SetRxRateLimiter() error {
 	endpoint.RxRateLimiter = true
 	return nil
 }
 
-func (endpoint *MacvlanEndpoint) GetTxRateLimiter() bool {
+func (endpoint *BridgedMacvlanEndpoint) GetTxRateLimiter() bool {
 	return endpoint.TxRateLimiter
 }
 
-func (endpoint *MacvlanEndpoint) SetTxRateLimiter() error {
+func (endpoint *BridgedMacvlanEndpoint) SetTxRateLimiter() error {
 	endpoint.TxRateLimiter = true
 	return nil
 }

src/runtime/virtcontainers/bridgedmacvlan_endpoint_test.go

@@ -12,10 +12,10 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-func TestCreateMacvlanEndpoint(t *testing.T) {
+func TestCreateBridgedMacvlanEndpoint(t *testing.T) {
 	macAddr := net.HardwareAddr{0x02, 0x00, 0xCA, 0xFE, 0x00, 0x04}
 
-	expected := &MacvlanEndpoint{
+	expected := &BridgedMacvlanEndpoint{
 		NetPair: NetworkInterfacePair{
 			TapInterface: TapInterface{
 				ID:   "uniqueTestID-4",
@@ -30,10 +30,10 @@ func TestCreateMacvlanEndpoint(t *testing.T) {
 			},
 			NetInterworkingModel: DefaultNetInterworkingModel,
 		},
-		EndpointType: MacvlanEndpointType,
+		EndpointType: BridgedMacvlanEndpointType,
 	}
 
-	result, err := createMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel)
+	result, err := createBridgedMacvlanNetworkEndpoint(4, "", DefaultNetInterworkingModel)
 	assert.NoError(t, err)
 
 	// the resulting ID  will be random - so let's overwrite to test the rest of the flow

src/runtime/virtcontainers/clh.go

@@ -20,16 +20,16 @@ import (
 	"time"
 
 	"github.com/containerd/console"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	chclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cloud-hypervisor/client"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -157,6 +157,7 @@ func (s *CloudHypervisorState) reset() {
 }
 
 type cloudHypervisor struct {
+	store     persistapi.PersistDriver
 	console   console.Console
 	virtiofsd Virtiofsd
 	APIClient clhClient
@@ -225,7 +226,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, networkNS N
 		clh.Logger().WithField("function", "CreateVM").Info("Sandbox already exist, loading from state")
 		clh.virtiofsd = &virtiofsd{
 			PID:        clh.state.VirtiofsdPID,
-			sourcePath: hypervisorConfig.SharedPath,
+			sourcePath: filepath.Join(getSharePath(clh.id)),
 			debug:      clh.config.Debug,
 			socketPath: virtiofsdSocketPath,
 		}
@@ -341,7 +342,7 @@ func (clh *cloudHypervisor) CreateVM(ctx context.Context, id string, networkNS N
 
 	clh.virtiofsd = &virtiofsd{
 		path:       clh.config.VirtioFSDaemon,
-		sourcePath: filepath.Join(GetSharePath(clh.id)),
+		sourcePath: filepath.Join(getSharePath(clh.id)),
 		socketPath: virtiofsdSocketPath,
 		extraArgs:  clh.config.VirtioFSExtraArgs,
 		debug:      clh.config.Debug,
@@ -373,7 +374,7 @@ func (clh *cloudHypervisor) StartVM(ctx context.Context, timeout int) error {
 
 	clh.Logger().WithField("function", "StartVM").Info("starting Sandbox")
 
-	vmPath := filepath.Join(clh.config.VMStorePath, clh.id)
+	vmPath := filepath.Join(clh.store.RunVMStoragePath(), clh.id)
 	err := os.MkdirAll(vmPath, DirMode)
 	if err != nil {
 		return err
@@ -746,7 +747,7 @@ func (clh *cloudHypervisor) toGrpc(ctx context.Context) ([]byte, error) {
 	return nil, errors.New("cloudHypervisor is not supported by VM cache")
 }
 
-func (clh *cloudHypervisor) Save() (s hv.HypervisorState) {
+func (clh *cloudHypervisor) Save() (s persistapi.HypervisorState) {
 	s.Pid = clh.state.PID
 	s.Type = string(ClhHypervisor)
 	s.VirtiofsdPid = clh.state.VirtiofsdPID
@@ -754,24 +755,18 @@ func (clh *cloudHypervisor) Save() (s hv.HypervisorState) {
 	return
 }
 
-func (clh *cloudHypervisor) Load(s hv.HypervisorState) {
+func (clh *cloudHypervisor) Load(s persistapi.HypervisorState) {
 	clh.state.PID = s.Pid
 	clh.state.VirtiofsdPID = s.VirtiofsdPid
 	clh.state.apiSocket = s.APISocket
 }
 
-// Check is the implementation of Check from the Hypervisor interface.
-// Check if the VMM API is working.
-
 func (clh *cloudHypervisor) Check() error {
-	// Use a long timeout to check if the VMM is running:
-	// Check is used by the monitor thread(a background thread). If the
-	// monitor thread calls Check() during the Container boot, it will take
-	// longer than usual specially if there is a hot-plug request in progress.
-	running, err := clh.isClhRunning(10)
-	if !running {
-		return fmt.Errorf("clh is not running: %s", err)
-	}
+	cl := clh.client()
+	ctx, cancel := context.WithTimeout(context.Background(), clhAPITimeout*time.Second)
+	defer cancel()
+
+	_, _, err := cl.VmmPingGet(ctx)
 	return err
 }
 
@@ -813,7 +808,7 @@ func (clh *cloudHypervisor) AddDevice(ctx context.Context, devInfo interface{},
 //###########################################################################
 
 func (clh *cloudHypervisor) Logger() *log.Entry {
-	return hvLogger.WithField("subsystem", "cloudHypervisor")
+	return virtLog.WithField("subsystem", "cloudHypervisor")
 }
 
 // Adds all capabilities supported by cloudHypervisor implementation of hypervisor interface
@@ -892,15 +887,15 @@ func (clh *cloudHypervisor) GenerateSocket(id string) (interface{}, error) {
 }
 
 func (clh *cloudHypervisor) virtioFsSocketPath(id string) (string, error) {
-	return utils.BuildSocketPath(clh.config.VMStorePath, id, virtioFsSocket)
+	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, virtioFsSocket)
 }
 
 func (clh *cloudHypervisor) vsockSocketPath(id string) (string, error) {
-	return utils.BuildSocketPath(clh.config.VMStorePath, id, clhSocket)
+	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhSocket)
 }
 
 func (clh *cloudHypervisor) apiSocketPath(id string) (string, error) {
-	return utils.BuildSocketPath(clh.config.VMStorePath, id, clhAPISocket)
+	return utils.BuildSocketPath(clh.store.RunVMStoragePath(), id, clhAPISocket)
 }
 
 func (clh *cloudHypervisor) waitVMM(timeout uint) error {
@@ -1039,6 +1034,8 @@ func (clh *cloudHypervisor) isClhRunning(timeout uint) (bool, error) {
 
 	pid := clh.state.PID
 
+	// Check if clh process is running, in case it is not, let's
+	// return from here.
 	if err := syscall.Kill(pid, syscall.Signal(0)); err != nil {
 		return false, nil
 	}
@@ -1051,8 +1048,6 @@ func (clh *cloudHypervisor) isClhRunning(timeout uint) (bool, error) {
 		_, _, err := cl.VmmPingGet(ctx)
 		if err == nil {
 			return true, nil
-		} else {
-			clh.Logger().WithError(err).Warning("clh.VmmPingGet API call failed")
 		}
 
 		if time.Since(timeStart).Seconds() > float64(timeout) {
@@ -1212,7 +1207,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error {
 	}
 
 	// Cleanup vm path
-	dir := filepath.Join(clh.config.VMStorePath, clh.id)
+	dir := filepath.Join(clh.store.RunVMStoragePath(), clh.id)
 
 	// If it's a symlink, remove both dir and the target.
 	link, err := filepath.EvalSymlinks(dir)
@@ -1241,7 +1236,7 @@ func (clh *cloudHypervisor) cleanupVM(force bool) error {
 	}
 
 	if clh.config.VMid != "" {
-		dir = filepath.Join(clh.config.VMStorePath, clh.config.VMid)
+		dir = filepath.Join(clh.store.RunStoragePath(), clh.config.VMid)
 		if err := os.RemoveAll(dir); err != nil {
 			if !force {
 				return err
@@ -1271,3 +1266,6 @@ func (clh *cloudHypervisor) vmInfo() (chclient.VmInfo, error) {
 func (clh *cloudHypervisor) IsRateLimiterBuiltin() bool {
 	return false
 }
+
+func (clh *cloudHypervisor) setSandbox(sandbox *Sandbox) {
+}

src/runtime/virtcontainers/clh_test.go

@@ -203,10 +203,7 @@ func TestCloudHypervisorCleanupVM(t *testing.T) {
 	assert.NoError(err, "persist.GetDriver() unexpected error")
 
 	clh := &cloudHypervisor{
-		config: HypervisorConfig{
-			VMStorePath:  store.RunVMStoragePath(),
-			RunStorePath: store.RunStoragePath(),
-		},
+		store: store,
 	}
 
 	err = clh.cleanupVM(true)
@@ -217,7 +214,7 @@ func TestCloudHypervisorCleanupVM(t *testing.T) {
 	err = clh.cleanupVM(true)
 	assert.NoError(err, "persist.GetDriver() unexpected error")
 
-	dir := filepath.Join(store.RunVMStoragePath(), clh.id)
+	dir := filepath.Join(clh.store.RunVMStoragePath(), clh.id)
 	os.MkdirAll(dir, os.ModePerm)
 
 	err = clh.cleanupVM(false)
@@ -238,11 +235,9 @@ func TestClhCreateVM(t *testing.T) {
 	store, err := persist.GetDriver()
 	assert.NoError(err)
 
-	clhConfig.VMStorePath = store.RunVMStoragePath()
-	clhConfig.RunStorePath = store.RunStoragePath()
-
 	clh := &cloudHypervisor{
 		config: clhConfig,
+		store:  store,
 	}
 
 	sandbox := &Sandbox{
@@ -266,13 +261,11 @@ func TestClooudHypervisorStartSandbox(t *testing.T) {
 	store, err := persist.GetDriver()
 	assert.NoError(err)
 
-	clhConfig.VMStorePath = store.RunVMStoragePath()
-	clhConfig.RunStorePath = store.RunStoragePath()
-
 	clh := &cloudHypervisor{
 		config:    clhConfig,
 		APIClient: &clhClientMock{},
 		virtiofsd: &virtiofsdMock{},
+		store:     store,
 	}
 
 	err = clh.StartVM(context.Background(), 10)
@@ -386,11 +379,6 @@ func TestClhGenerateSocket(t *testing.T) {
 	clh, ok := hypervisor.(*cloudHypervisor)
 	assert.True(ok)
 
-	clh.config = HypervisorConfig{
-		VMStorePath:  "/foo",
-		RunStorePath: "/bar",
-	}
-
 	clh.addVSock(1, "path")
 
 	s, err := clh.GenerateSocket("c")
@@ -403,7 +391,7 @@ func TestClhGenerateSocket(t *testing.T) {
 	assert.NotEmpty(hvsock.UdsPath)
 
 	// Path must be absolute
-	assert.True(strings.HasPrefix(hvsock.UdsPath, "/"), "failed: socket path: %s", hvsock.UdsPath)
+	assert.True(strings.HasPrefix(hvsock.UdsPath, "/"))
 
 	assert.NotZero(hvsock.Port)
 }

src/runtime/virtcontainers/container.go

@@ -1,6 +1,4 @@
-//go:build linux
 // +build linux
-
 // Copyright (c) 2016 Intel Corporation
 // Copyright (c) 2014,2015,2016,2017 Docker, Inc.
 // SPDX-License-Identifier: Apache-2.0
@@ -24,8 +22,8 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 
 	specs "github.com/opencontainers/runtime-spec/specs-go"

src/runtime/virtcontainers/device/config/config.go

@@ -15,7 +15,7 @@ import (
 	"strings"
 
 	"github.com/go-ini/ini"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"golang.org/x/sys/unix"
 )
 

src/runtime/virtcontainers/documentation/api/1.0/api.md

@@ -366,6 +366,7 @@ type NetworkConfig struct {
 	NetNSPath         string
 	NetNsCreated      bool
 	DisableNewNetNs   bool
+	NetmonConfig      NetmonConfig
 	InterworkingModel NetInterworkingModel
 }
 ```

src/runtime/virtcontainers/endpoint.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 )
 
 // Endpoint represents a physical or virtual network interface.
@@ -51,8 +51,8 @@ const (
 	// VhostUserEndpointType is the vhostuser network interface.
 	VhostUserEndpointType EndpointType = "vhost-user"
 
-	// MacvlanEndpointType is macvlan network interface.
-	MacvlanEndpointType EndpointType = "macvlan"
+	// BridgedMacvlanEndpointType is macvlan network interface.
+	BridgedMacvlanEndpointType EndpointType = "macvlan"
 
 	// MacvtapEndpointType is macvtap network interface.
 	MacvtapEndpointType EndpointType = "macvtap"
@@ -80,7 +80,7 @@ func (endpointType *EndpointType) Set(value string) error {
 		*endpointType = VhostUserEndpointType
 		return nil
 	case "macvlan":
-		*endpointType = MacvlanEndpointType
+		*endpointType = BridgedMacvlanEndpointType
 		return nil
 	case "macvtap":
 		*endpointType = MacvtapEndpointType
@@ -108,8 +108,8 @@ func (endpointType *EndpointType) String() string {
 		return string(VethEndpointType)
 	case VhostUserEndpointType:
 		return string(VhostUserEndpointType)
-	case MacvlanEndpointType:
-		return string(MacvlanEndpointType)
+	case BridgedMacvlanEndpointType:
+		return string(BridgedMacvlanEndpointType)
 	case MacvtapEndpointType:
 		return string(MacvtapEndpointType)
 	case TapEndpointType:

src/runtime/virtcontainers/endpoint_test.go

@@ -35,8 +35,8 @@ func TestVhostUserEndpointTypeSet(t *testing.T) {
 	testEndpointTypeSet(t, "vhost-user", VhostUserEndpointType)
 }
 
-func TestMacvlanEndpointTypeSet(t *testing.T) {
-	testEndpointTypeSet(t, "macvlan", MacvlanEndpointType)
+func TestBridgedMacvlanEndpointTypeSet(t *testing.T) {
+	testEndpointTypeSet(t, "macvlan", BridgedMacvlanEndpointType)
 }
 
 func TestMacvtapEndpointTypeSet(t *testing.T) {
@@ -69,9 +69,9 @@ func TestVhostUserEndpointTypeString(t *testing.T) {
 	testEndpointTypeString(t, &endpointType, string(VhostUserEndpointType))
 }
 
-func TestMacvlanEndpointTypeString(t *testing.T) {
-	endpointType := MacvlanEndpointType
-	testEndpointTypeString(t, &endpointType, string(MacvlanEndpointType))
+func TestBridgedMacvlanEndpointTypeString(t *testing.T) {
+	endpointType := BridgedMacvlanEndpointType
+	testEndpointTypeString(t, &endpointType, string(BridgedMacvlanEndpointType))
 }
 
 func TestMacvtapEndpointTypeString(t *testing.T) {

src/runtime/virtcontainers/fc.go

@@ -22,9 +22,9 @@ import (
 	"syscall"
 	"time"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client"
 	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"
 	ops "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/operations"
@@ -1226,13 +1226,13 @@ func (fc *firecracker) toGrpc(ctx context.Context) ([]byte, error) {
 	return nil, errors.New("firecracker is not supported by VM cache")
 }
 
-func (fc *firecracker) Save() (s hv.HypervisorState) {
+func (fc *firecracker) Save() (s persistapi.HypervisorState) {
 	s.Pid = fc.info.PID
 	s.Type = string(FirecrackerHypervisor)
 	return
 }
 
-func (fc *firecracker) Load(s hv.HypervisorState) {
+func (fc *firecracker) Load(s persistapi.HypervisorState) {
 	fc.info.PID = s.Pid
 }
 
@@ -1274,3 +1274,6 @@ func revertBytes(num uint64) uint64 {
 	}
 	return 1024*revertBytes(a) + b
 }
+
+func (fc *firecracker) setSandbox(sandbox *Sandbox) {
+}

src/runtime/virtcontainers/hypervisor.go

@@ -14,12 +14,11 @@ import (
 	"strconv"
 	"strings"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
-
-	"github.com/sirupsen/logrus"
 )
 
 // HypervisorType describes an hypervisor type.
@@ -47,10 +46,14 @@ const (
 
 	// MockHypervisor is a mock hypervisor for testing purposes
 	MockHypervisor HypervisorType = "mock"
+)
 
+const (
 	procMemInfo = "/proc/meminfo"
 	procCPUInfo = "/proc/cpuinfo"
+)
 
+const (
 	defaultVCPUs = 1
 	// 2 GiB
 	defaultMemSzMiB = 2048
@@ -71,10 +74,6 @@ const (
 	MinHypervisorMemory = 256
 )
 
-var (
-	hvLogger = logrus.WithField("source", "virtcontainers/hypervisor")
-)
-
 // In some architectures the maximum number of vCPUs depends on the number of physical cores.
 var defaultMaxQemuVCPUs = MaxQemuVCPUs()
 
@@ -145,12 +144,6 @@ type MemoryDevice struct {
 	Probe  bool
 }
 
-// SetHypervisorLogger sets up a logger for the hypervisor part of this pkg
-func SetHypervisorLogger(logger *logrus.Entry) {
-	fields := hvLogger.Data
-	hvLogger = logger.WithFields(fields)
-}
-
 // Set sets an hypervisor type based on the input string.
 func (hType *HypervisorType) Set(value string) error {
 	switch value {
@@ -192,18 +185,28 @@ func (hType *HypervisorType) String() string {
 	}
 }
 
-// NewHypervisor returns an hypervisor from a hypervisor type.
+// NewHypervisor returns an hypervisor from and hypervisor type.
 func NewHypervisor(hType HypervisorType) (Hypervisor, error) {
+	store, err := persist.GetDriver()
+	if err != nil {
+		return nil, err
+	}
 
 	switch hType {
 	case QemuHypervisor:
-		return &qemu{}, nil
+		return &qemu{
+			store: store,
+		}, nil
 	case FirecrackerHypervisor:
 		return &firecracker{}, nil
 	case AcrnHypervisor:
-		return &Acrn{}, nil
+		return &Acrn{
+			store: store,
+		}, nil
 	case ClhHypervisor:
-		return &cloudHypervisor{}, nil
+		return &cloudHypervisor{
+			store: store,
+		}, nil
 	case MockHypervisor:
 		return &mockHypervisor{}, nil
 	default:
@@ -312,19 +315,13 @@ type HypervisorConfig struct {
 	EntropySource string
 
 	// Shared file system type:
-	//   - virtio-9p
-	//   - virtio-fs (default)
+	//   - virtio-9p (default)
+	//   - virtio-fs
 	SharedFS string
 
-	// Path for filesystem sharing
-	SharedPath string
-
 	// VirtioFSDaemon is the virtio-fs vhost-user daemon path
 	VirtioFSDaemon string
 
-	// VirtioFSCache cache mode for fs version cache or "none"
-	VirtioFSCache string
-
 	// File based memory backend root directory
 	FileBackedMemRootDir string
 
@@ -342,15 +339,12 @@ type HypervisorConfig struct {
 	// VMid is "" if the hypervisor is not created by the factory.
 	VMid string
 
-	// VMStorePath is the location on disk where VM information will persist
-	VMStorePath string
-
-	// VMStorePath is the location on disk where runtime information will persist
-	RunStorePath string
-
 	// SELinux label for the VM
 	SELinuxProcessLabel string
 
+	// VirtioFSCache cache mode for fs version cache or "none"
+	VirtioFSCache string
+
 	// HypervisorPathList is the list of hypervisor paths names allowed in annotations
 	HypervisorPathList []string
 
@@ -612,7 +606,7 @@ func (conf *HypervisorConfig) AddCustomAsset(a *types.Asset) error {
 		return fmt.Errorf("Invalid %s at %s", a.Type(), a.Path())
 	}
 
-	hvLogger.Debugf("Using custom %v asset %s", a.Type(), a.Path())
+	virtLog.Debugf("Using custom %v asset %s", a.Type(), a.Path())
 
 	if conf.customAssets == nil {
 		conf.customAssets = make(map[types.AssetType]*types.Asset)
@@ -880,7 +874,7 @@ func RunningOnVMM(cpuInfoPath string) (bool, error) {
 		return flags["hypervisor"], nil
 	}
 
-	hvLogger.WithField("arch", runtime.GOARCH).Info("Unable to know if the system is running inside a VM")
+	virtLog.WithField("arch", runtime.GOARCH).Info("Unable to know if the system is running inside a VM")
 	return false, nil
 }
 
@@ -937,12 +931,14 @@ type Hypervisor interface {
 	toGrpc(ctx context.Context) ([]byte, error)
 	Check() error
 
-	Save() hv.HypervisorState
-	Load(hv.HypervisorState)
+	Save() persistapi.HypervisorState
+	Load(persistapi.HypervisorState)
 
 	// generate the socket to communicate the host and guest
 	GenerateSocket(id string) (interface{}, error)
 
 	// check if hypervisor supports built-in rate limiter.
 	IsRateLimiterBuiltin() bool
+
+	setSandbox(sandbox *Sandbox)
 }

src/runtime/virtcontainers/ipvlan_endpoint.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/containernetworking/plugins/pkg/ns"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 )
 
 var ipvlanTrace = getNetworkTrace(IPVlanEndpointType)

src/runtime/virtcontainers/kata_agent.go

@@ -19,7 +19,6 @@ import (
 	"time"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@@ -29,8 +28,9 @@ import (
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	vccgroups "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cgroups"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 
 	"github.com/gogo/protobuf/proto"
 	"github.com/opencontainers/runtime-spec/specs-go"
@@ -134,6 +134,8 @@ const (
 	grpcMemHotplugByProbeRequest = "grpc.MemHotplugByProbeRequest"
 	grpcCopyFileRequest          = "grpc.CopyFileRequest"
 	grpcSetGuestDateTimeRequest  = "grpc.SetGuestDateTimeRequest"
+	grpcStartTracingRequest      = "grpc.StartTracingRequest"
+	grpcStopTracingRequest       = "grpc.StopTracingRequest"
 	grpcGetOOMEventRequest       = "grpc.GetOOMEventRequest"
 	grpcGetMetricsRequest        = "grpc.GetMetricsRequest"
 	grpcAddSwapRequest           = "grpc.AddSwapRequest"
@@ -162,7 +164,7 @@ var kataHostSharedDir = func() string {
 // 2. /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ is bind mounted readonly to /run/kata-containers/shared/sandboxes/$sbx_id/shared/, so guest cannot modify it
 //
 // 3. host-guest shared files/directories are mounted one-level under /run/kata-containers/shared/sandboxes/$sbx_id/mounts/ and thus present to guest at one level under /run/kata-containers/shared/sandboxes/$sbx_id/shared/
-func GetSharePath(id string) string {
+func getSharePath(id string) string {
 	return filepath.Join(kataHostSharedDir(), id, "shared")
 }
 
@@ -242,8 +244,9 @@ type kataAgent struct {
 
 	dialTimout uint32
 
-	keepConn bool
-	dead     bool
+	keepConn       bool
+	dynamicTracing bool
+	dead           bool
 }
 
 func (k *kataAgent) Logger() *logrus.Entry {
@@ -356,7 +359,7 @@ func (k *kataAgent) setupSandboxBindMounts(ctx context.Context, sandbox *Sandbox
 
 	// Create subdirectory in host shared path for sandbox mounts
 	sandboxMountDir := filepath.Join(getMountPath(sandbox.id), sandboxMountsDir)
-	sandboxShareDir := filepath.Join(GetSharePath(sandbox.id), sandboxMountsDir)
+	sandboxShareDir := filepath.Join(getSharePath(sandbox.id), sandboxMountsDir)
 	if err := os.MkdirAll(sandboxMountDir, DirMode); err != nil {
 		return fmt.Errorf("Creating sandbox shared mount directory: %v: %w", sandboxMountDir, err)
 	}
@@ -473,7 +476,7 @@ func (k *kataAgent) setupSharedPath(ctx context.Context, sandbox *Sandbox) (err
 	defer span.End()
 
 	// create shared path structure
-	sharePath := GetSharePath(sandbox.id)
+	sharePath := getSharePath(sandbox.id)
 	mountPath := getMountPath(sandbox.id)
 	if err := os.MkdirAll(sharePath, sharedDirMode); err != nil {
 		return err
@@ -509,7 +512,7 @@ func (k *kataAgent) createSandbox(ctx context.Context, sandbox *Sandbox) error {
 	if err := k.setupSharedPath(ctx, sandbox); err != nil {
 		return err
 	}
-	return k.configure(ctx, sandbox.hypervisor, sandbox.id, GetSharePath(sandbox.id), sandbox.config.AgentConfig)
+	return k.configure(ctx, sandbox.hypervisor, sandbox.id, getSharePath(sandbox.id), sandbox.config.AgentConfig)
 }
 
 func cmdToKataProcess(cmd types.Cmd) (process *grpc.Process, err error) {
@@ -818,6 +821,13 @@ func (k *kataAgent) startSandbox(ctx context.Context, sandbox *Sandbox) error {
 		return err
 	}
 
+	if k.dynamicTracing {
+		_, err = k.sendReq(ctx, &grpc.StartTracingRequest{})
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -917,6 +927,13 @@ func (k *kataAgent) stopSandbox(ctx context.Context, sandbox *Sandbox) error {
 		return err
 	}
 
+	if k.dynamicTracing {
+		_, err := k.sendReq(ctx, &grpc.StopTracingRequest{})
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -2009,6 +2026,12 @@ func (k *kataAgent) installReqFunc(c *kataclient.AgentClient) {
 	k.reqHandlers[grpcSetGuestDateTimeRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.SetGuestDateTime(ctx, req.(*grpc.SetGuestDateTimeRequest))
 	}
+	k.reqHandlers[grpcStartTracingRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.StartTracing(ctx, req.(*grpc.StartTracingRequest))
+	}
+	k.reqHandlers[grpcStopTracingRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
+		return k.client.AgentServiceClient.StopTracing(ctx, req.(*grpc.StopTracingRequest))
+	}
 	k.reqHandlers[grpcGetOOMEventRequest] = func(ctx context.Context, req interface{}) (interface{}, error) {
 		return k.client.AgentServiceClient.GetOOMEvent(ctx, req.(*grpc.GetOOMEventRequest))
 	}
@@ -2198,7 +2221,7 @@ func (k *kataAgent) cleanup(ctx context.Context, s *Sandbox) {
 	}
 
 	// Unmount shared path
-	path := GetSharePath(s.id)
+	path := getSharePath(s.id)
 	k.Logger().WithField("path", path).Infof("Cleanup agent")
 	if err := syscall.Unmount(path, syscall.MNT_DETACH|UmountNoFollow); err != nil {
 		k.Logger().WithError(err).Errorf("failed to unmount vm share path %s", path)

src/runtime/virtcontainers/kata_agent_test.go

@@ -31,8 +31,8 @@ import (
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/mock"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )
 
 var (
@@ -1158,7 +1158,7 @@ func TestSandboxBindMount(t *testing.T) {
 	assert.Nil(err)
 	defer os.RemoveAll(dir)
 
-	sharePath := GetSharePath(sandbox.id)
+	sharePath := getSharePath(sandbox.id)
 	mountPath := getMountPath(sandbox.id)
 
 	err = os.MkdirAll(sharePath, DirMode)

src/runtime/virtcontainers/macvtap_endpoint.go

@@ -11,7 +11,7 @@ import (
 	"os"
 
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 )
 
 var macvtapTrace = getNetworkTrace(MacvtapEndpointType)

src/runtime/virtcontainers/mock_agent.go

@@ -12,8 +12,8 @@ import (
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/net/context"
 )

src/runtime/virtcontainers/mock_hypervisor.go

@@ -10,7 +10,7 @@ import (
 	"errors"
 	"os"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
+	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )
 
@@ -130,11 +130,11 @@ func (m *mockHypervisor) toGrpc(ctx context.Context) ([]byte, error) {
 	return nil, errors.New("mockHypervisor is not supported by VM cache")
 }
 
-func (m *mockHypervisor) Save() (s hv.HypervisorState) {
+func (m *mockHypervisor) Save() (s persistapi.HypervisorState) {
 	return
 }
 
-func (m *mockHypervisor) Load(s hv.HypervisorState) {}
+func (m *mockHypervisor) Load(s persistapi.HypervisorState) {}
 
 func (m *mockHypervisor) Check() error {
 	return nil
@@ -149,3 +149,6 @@ func (m *mockHypervisor) GenerateSocket(id string) (interface{}, error) {
 func (m *mockHypervisor) IsRateLimiterBuiltin() bool {
 	return false
 }
+
+func (m *mockHypervisor) setSandbox(sandbox *Sandbox) {
+}

src/runtime/virtcontainers/monitor.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	defaultCheckInterval = 5 * time.Second
+	defaultCheckInterval = 1 * time.Second
 	watcherChannelSize   = 128
 )
 

src/runtime/virtcontainers/netmon.go

@@ -0,0 +1,96 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"fmt"
+	"os/exec"
+	"syscall"
+
+	"github.com/sirupsen/logrus"
+)
+
+// NetmonConfig is the structure providing specific configuration
+// for the network monitor.
+type NetmonConfig struct {
+	Path   string
+	Debug  bool
+	Enable bool
+}
+
+// netmonParams is the structure providing specific parameters needed
+// for the execution of the network monitor binary.
+type netmonParams struct {
+	netmonPath string
+	logLevel   string
+	runtime    string
+	sandboxID  string
+	debug      bool
+}
+
+func netmonLogger() *logrus.Entry {
+	return virtLog.WithField("subsystem", "netmon")
+}
+
+func prepareNetMonParams(params netmonParams) ([]string, error) {
+	if params.netmonPath == "" {
+		return []string{}, fmt.Errorf("Netmon path is empty")
+	}
+	if params.runtime == "" {
+		return []string{}, fmt.Errorf("Netmon runtime path is empty")
+	}
+	if params.sandboxID == "" {
+		return []string{}, fmt.Errorf("Netmon sandbox ID is empty")
+	}
+
+	args := []string{params.netmonPath,
+		"-r", params.runtime,
+		"-s", params.sandboxID,
+	}
+
+	if params.debug {
+		args = append(args, "-d")
+	}
+	if params.logLevel != "" {
+		args = append(args, []string{"-log", params.logLevel}...)
+	}
+
+	return args, nil
+}
+
+func startNetmon(params netmonParams) (int, error) {
+	args, err := prepareNetMonParams(params)
+	if err != nil {
+		return -1, err
+	}
+
+	cmd := exec.Command(args[0], args[1:]...)
+	if err := cmd.Start(); err != nil {
+		return -1, err
+	}
+
+	return cmd.Process.Pid, nil
+}
+
+func stopNetmon(pid int) error {
+	if pid <= 0 {
+		return nil
+	}
+
+	sig := syscall.SIGKILL
+
+	netmonLogger().WithFields(
+		logrus.Fields{
+			"netmon-pid":    pid,
+			"netmon-signal": sig,
+		}).Info("Stopping netmon")
+
+	if err := syscall.Kill(pid, sig); err != nil && err != syscall.ESRCH {
+		return err
+	}
+
+	return nil
+}

src/runtime/virtcontainers/netmon_test.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2018 Intel Corporation
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package virtcontainers
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+const (
+	testNetmonPath  = "/foo/bar/netmon"
+	testRuntimePath = "/foo/bar/runtime"
+)
+
+func TestNetmonLogger(t *testing.T) {
+	got := netmonLogger()
+	expected := virtLog.WithField("subsystem", "netmon")
+	assert.True(t, reflect.DeepEqual(expected, got),
+		"Got %+v\nExpected %+v", got, expected)
+}
+
+func TestPrepareNetMonParams(t *testing.T) {
+	// Empty netmon path
+	params := netmonParams{}
+	got, err := prepareNetMonParams(params)
+	assert.NotNil(t, err)
+	assert.Equal(t, got, []string{})
+
+	// Empty runtime path
+	params.netmonPath = testNetmonPath
+	got, err = prepareNetMonParams(params)
+	assert.NotNil(t, err)
+	assert.Equal(t, got, []string{})
+
+	// Empty sandbox ID
+	params.runtime = testRuntimePath
+	got, err = prepareNetMonParams(params)
+	assert.NotNil(t, err)
+	assert.Equal(t, got, []string{})
+
+	// Successful case
+	params.sandboxID = testSandboxID
+	got, err = prepareNetMonParams(params)
+	assert.Nil(t, err)
+	expected := []string{testNetmonPath,
+		"-r", testRuntimePath,
+		"-s", testSandboxID}
+	assert.True(t, reflect.DeepEqual(expected, got),
+		"Got %+v\nExpected %+v", got, expected)
+}
+
+func TestStopNetmon(t *testing.T) {
+	pid := -1
+	err := stopNetmon(pid)
+	assert.Nil(t, err)
+}

src/runtime/virtcontainers/network.go

@@ -8,6 +8,7 @@ package virtcontainers
 import (
 	"context"
 	cryptoRand "crypto/rand"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"net"
@@ -25,9 +26,8 @@ import (
 	"golang.org/x/sys/unix"
 
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/rootless"
+	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/uuid"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -178,6 +178,7 @@ type NetworkInterfacePair struct {
 // NetworkConfig is the network configuration related to a network.
 type NetworkConfig struct {
 	NetNSPath         string
+	NetmonConfig      NetmonConfig
 	InterworkingModel NetInterworkingModel
 	NetNsCreated      bool
 	DisableNewNetNs   bool
@@ -192,6 +193,133 @@ type NetworkNamespace struct {
 	NetNsPath    string
 	Endpoints    []Endpoint
 	NetNsCreated bool
+	NetmonPID    int
+}
+
+// TypedJSONEndpoint is used as an intermediate representation for
+// marshalling and unmarshalling Endpoint objects.
+type TypedJSONEndpoint struct {
+	Type EndpointType
+	Data json.RawMessage
+}
+
+// MarshalJSON is the custom NetworkNamespace JSON marshalling routine.
+// This is needed to properly marshall Endpoints array.
+func (n NetworkNamespace) MarshalJSON() ([]byte, error) {
+	// We need a shadow structure in order to prevent json from
+	// entering a recursive loop when only calling json.Marshal().
+	type shadow struct {
+		NetNsPath    string
+		Endpoints    []TypedJSONEndpoint
+		NetNsCreated bool
+	}
+
+	s := &shadow{
+		NetNsPath:    n.NetNsPath,
+		NetNsCreated: n.NetNsCreated,
+	}
+
+	var typedEndpoints []TypedJSONEndpoint
+	for _, endpoint := range n.Endpoints {
+		tempJSON, _ := json.Marshal(endpoint)
+
+		t := TypedJSONEndpoint{
+			Type: endpoint.Type(),
+			Data: tempJSON,
+		}
+
+		typedEndpoints = append(typedEndpoints, t)
+	}
+
+	s.Endpoints = typedEndpoints
+
+	b, err := json.Marshal(s)
+	return b, err
+}
+
+func generateEndpoints(typedEndpoints []TypedJSONEndpoint) ([]Endpoint, error) {
+	var endpoints []Endpoint
+
+	for _, e := range typedEndpoints {
+		var endpointInf Endpoint
+		switch e.Type {
+		case PhysicalEndpointType:
+			var endpoint PhysicalEndpoint
+			endpointInf = &endpoint
+
+		case VethEndpointType:
+			var endpoint VethEndpoint
+			endpointInf = &endpoint
+
+		case VhostUserEndpointType:
+			var endpoint VhostUserEndpoint
+			endpointInf = &endpoint
+
+		case BridgedMacvlanEndpointType:
+			var endpoint BridgedMacvlanEndpoint
+			endpointInf = &endpoint
+
+		case MacvtapEndpointType:
+			var endpoint MacvtapEndpoint
+			endpointInf = &endpoint
+
+		case TapEndpointType:
+			var endpoint TapEndpoint
+			endpointInf = &endpoint
+
+		case IPVlanEndpointType:
+			var endpoint IPVlanEndpoint
+			endpointInf = &endpoint
+
+		case TuntapEndpointType:
+			var endpoint TuntapEndpoint
+			endpointInf = &endpoint
+
+		default:
+			networkLogger().WithField("endpoint-type", e.Type).Error("Ignoring unknown endpoint type")
+		}
+
+		err := json.Unmarshal(e.Data, endpointInf)
+		if err != nil {
+			return nil, err
+		}
+
+		endpoints = append(endpoints, endpointInf)
+		networkLogger().WithFields(logrus.Fields{
+			"endpoint":      endpointInf,
+			"endpoint-type": e.Type,
+		}).Info("endpoint unmarshalled")
+	}
+	return endpoints, nil
+}
+
+// UnmarshalJSON is the custom NetworkNamespace unmarshalling routine.
+// This is needed for unmarshalling the Endpoints interfaces array.
+func (n *NetworkNamespace) UnmarshalJSON(b []byte) error {
+	var s struct {
+		NetNsPath    string
+		Endpoints    json.RawMessage
+		NetNsCreated bool
+	}
+
+	if err := json.Unmarshal(b, &s); err != nil {
+		return err
+	}
+
+	(*n).NetNsPath = s.NetNsPath
+	(*n).NetNsCreated = s.NetNsCreated
+
+	var typedEndpoints []TypedJSONEndpoint
+	if err := json.Unmarshal([]byte(string(s.Endpoints)), &typedEndpoints); err != nil {
+		return err
+	}
+	endpoints, err := generateEndpoints(typedEndpoints)
+	if err != nil {
+		return err
+	}
+
+	(*n).Endpoints = endpoints
+	return nil
 }
 
 func createLink(netHandle *netlink.Handle, name string, expectedLink netlink.Link, queues int) (netlink.Link, []*os.File, error) {
@@ -249,7 +377,7 @@ func getLinkForEndpoint(endpoint Endpoint, netHandle *netlink.Handle) (netlink.L
 	switch ep := endpoint.(type) {
 	case *VethEndpoint:
 		link = &netlink.Veth{}
-	case *MacvlanEndpoint:
+	case *BridgedMacvlanEndpoint:
 		link = &netlink.Macvlan{}
 	case *IPVlanEndpoint:
 		link = &netlink.IPVlan{}
@@ -311,12 +439,7 @@ func xConnectVMNetwork(ctx context.Context, endpoint Endpoint, h Hypervisor) err
 		queues = int(h.HypervisorConfig().NumVCPUs)
 	}
 
-	var disableVhostNet bool
-	if rootless.IsRootless() {
-		disableVhostNet = true
-	} else {
-		disableVhostNet = h.HypervisorConfig().DisableVhostNet
-	}
+	disableVhostNet := h.HypervisorConfig().DisableVhostNet
 
 	if netPair.NetInterworkingModel == NetXConnectDefaultModel {
 		netPair.NetInterworkingModel = DefaultNetInterworkingModel
@@ -1125,7 +1248,7 @@ func createEndpoint(netInfo NetworkInfo, idx int, model NetInterworkingModel, li
 			endpoint, err = createVhostUserEndpoint(netInfo, socketPath)
 		} else if netInfo.Iface.Type == "macvlan" {
 			networkLogger().Infof("macvlan interface found")
-			endpoint, err = createMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
+			endpoint, err = createBridgedMacvlanNetworkEndpoint(idx, netInfo.Iface.Name, model)
 		} else if netInfo.Iface.Type == "macvtap" {
 			networkLogger().Infof("macvtap interface found")
 			endpoint, err = createMacvtapNetworkEndpoint(netInfo)
@@ -1327,7 +1450,7 @@ func (n *Network) Remove(ctx context.Context, ns *NetworkNamespace, hypervisor H
 func addRxRateLimiter(endpoint Endpoint, maxRate uint64) error {
 	var linkName string
 	switch ep := endpoint.(type) {
-	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *MacvlanEndpoint:
+	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *BridgedMacvlanEndpoint:
 		netPair := endpoint.NetworkPair()
 		linkName = netPair.TapInterface.TAPIface.Name
 	case *MacvtapEndpoint, *TapEndpoint:
@@ -1483,7 +1606,7 @@ func addTxRateLimiter(endpoint Endpoint, maxRate uint64) error {
 	var netPair *NetworkInterfacePair
 	var linkName string
 	switch ep := endpoint.(type) {
-	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *MacvlanEndpoint:
+	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *BridgedMacvlanEndpoint:
 		netPair = endpoint.NetworkPair()
 		switch netPair.NetInterworkingModel {
 		// For those endpoints we've already used tcfilter as their inter-networking model,
@@ -1556,7 +1679,7 @@ func removeHTBQdisc(linkName string) error {
 func removeRxRateLimiter(endpoint Endpoint, networkNSPath string) error {
 	var linkName string
 	switch ep := endpoint.(type) {
-	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *MacvlanEndpoint:
+	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *BridgedMacvlanEndpoint:
 		netPair := endpoint.NetworkPair()
 		linkName = netPair.TapInterface.TAPIface.Name
 	case *MacvtapEndpoint, *TapEndpoint:
@@ -1577,7 +1700,7 @@ func removeRxRateLimiter(endpoint Endpoint, networkNSPath string) error {
 func removeTxRateLimiter(endpoint Endpoint, networkNSPath string) error {
 	var linkName string
 	switch ep := endpoint.(type) {
-	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *MacvlanEndpoint:
+	case *VethEndpoint, *IPVlanEndpoint, *TuntapEndpoint, *BridgedMacvlanEndpoint:
 		netPair := endpoint.NetworkPair()
 		switch netPair.NetInterworkingModel {
 		case NetXConnectTCFilterModel:

src/runtime/virtcontainers/persist.go

@@ -8,7 +8,6 @@ package virtcontainers
 import (
 	"errors"
 
-	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
@@ -165,6 +164,7 @@ func (s *Sandbox) dumpAgent(ss *persistapi.SandboxState) {
 func (s *Sandbox) dumpNetwork(ss *persistapi.SandboxState) {
 	ss.Network = persistapi.NetworkInfo{
 		NetNsPath:    s.networkNS.NetNsPath,
+		NetmonPID:    s.networkNS.NetmonPID,
 		NetNsCreated: s.networkNS.NetNsCreated,
 	}
 	for _, e := range s.networkNS.Endpoints {
@@ -315,7 +315,7 @@ func (c *Container) loadContState(cs persistapi.ContainerState) {
 	}
 }
 
-func (s *Sandbox) loadHypervisor(hs hv.HypervisorState) {
+func (s *Sandbox) loadHypervisor(hs persistapi.HypervisorState) {
 	s.hypervisor.Load(hs)
 }
 
@@ -367,6 +367,7 @@ func (c *Container) loadContProcess(cs persistapi.ContainerState) {
 func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) {
 	s.networkNS = NetworkNamespace{
 		NetNsPath:    netInfo.NetNsPath,
+		NetmonPID:    netInfo.NetmonPID,
 		NetNsCreated: netInfo.NetNsCreated,
 	}
 
@@ -379,8 +380,8 @@ func (s *Sandbox) loadNetwork(netInfo persistapi.NetworkInfo) {
 			ep = &VethEndpoint{}
 		case VhostUserEndpointType:
 			ep = &VhostUserEndpoint{}
-		case MacvlanEndpointType:
-			ep = &MacvlanEndpoint{}
+		case BridgedMacvlanEndpointType:
+			ep = &BridgedMacvlanEndpoint{}
 		case MacvtapEndpointType:
 			ep = &MacvtapEndpoint{}
 		case TapEndpointType:

src/runtime/virtcontainers/persist/api/device.go

@@ -6,7 +6,7 @@
 
 package persistapi
 
-import vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+import vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 
 // ============= sandbox level resources =============
 

src/runtime/virtcontainers/persist/api/hypervisor.go

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0
 //
 
-package hypervisors
+package persistapi
 
 // Bridge is a bridge where devices can be hot plugged
 type Bridge struct {

src/runtime/virtcontainers/persist/api/network.go

@@ -7,7 +7,7 @@
 package persistapi
 
 import (
-	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
+	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/types"
 	"github.com/vishvananda/netlink"
 )
 
@@ -60,7 +60,7 @@ type TuntapEndpoint struct {
 	TuntapInterface TuntapInterface
 }
 
-type MacvlanEndpoint struct {
+type BridgedMacvlanEndpoint struct {
 	NetPair NetworkInterfacePair
 }
 
@@ -82,14 +82,14 @@ type VhostUserEndpoint struct {
 // NetworkEndpoint contains network interface information
 type NetworkEndpoint struct {
 	// One and only one of these below are not nil according to Type.
-	Physical  *PhysicalEndpoint  `json:",omitempty"`
-	Veth      *VethEndpoint      `json:",omitempty"`
-	VhostUser *VhostUserEndpoint `json:",omitempty"`
-	Macvlan   *MacvlanEndpoint   `json:",omitempty"`
-	Macvtap   *MacvtapEndpoint   `json:",omitempty"`
-	Tap       *TapEndpoint       `json:",omitempty"`
-	IPVlan    *IPVlanEndpoint    `json:",omitempty"`
-	Tuntap    *TuntapEndpoint    `json:",omitempty"`
+	Physical       *PhysicalEndpoint       `json:",omitempty"`
+	Veth           *VethEndpoint           `json:",omitempty"`
+	VhostUser      *VhostUserEndpoint      `json:",omitempty"`
+	BridgedMacvlan *BridgedMacvlanEndpoint `json:",omitempty"`
+	Macvtap        *MacvtapEndpoint        `json:",omitempty"`
+	Tap            *TapEndpoint            `json:",omitempty"`
+	IPVlan         *IPVlanEndpoint         `json:",omitempty"`
+	Tuntap         *TuntapEndpoint         `json:",omitempty"`
 
 	Type string
 }
@@ -98,5 +98,6 @@ type NetworkEndpoint struct {
 type NetworkInfo struct {
 	NetNsPath    string
 	Endpoints    []NetworkEndpoint
+	NetmonPID    int
 	NetNsCreated bool
 }