Merge pull request #4536 from Tim-0731-Hzt/runtime-rs-kata-main

runtime-rs: Merge Main into runtime-rs branch

.github/workflows/add-pr-sizing-label.yaml

@@ -33,6 +33,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.KATA_GITHUB_ACTIONS_PR_SIZE_TOKEN }}
         run: |
           pr=${{ github.event.number }}
+          # Removing man-db, workflow kept failing, fixes: #4480
+          sudo apt -y remove --purge man-db
           sudo apt -y install diffstat patchutils
 
           pr-add-size-label.sh -p "$pr"

.github/workflows/snap-release.yaml

@@ -19,6 +19,8 @@ jobs:
 
       - name: Build snap
         run: |
+          # Removing man-db, workflow kept failing, fixes: #4480
+          sudo apt -y remove --purge man-db
           sudo apt-get install -y git git-extras
           kata_url="https://github.com/kata-containers/kata-containers"
           latest_version=$(git ls-remote --tags ${kata_url}  | egrep -o "refs.*" | egrep -v "\-alpha|\-rc|{}" | egrep -o "[[:digit:]]+\.[[:digit:]]+\.[[:digit:]]+" | sort -V -r | head -1)

ci/darwin-test.sh

@@ -11,10 +11,10 @@ runtimedir=$cidir/../src/runtime
 
 build_working_packages() {
 	# working packages:
-	device_api=$runtimedir/virtcontainers/device/api
-	device_config=$runtimedir/virtcontainers/device/config
-	device_drivers=$runtimedir/virtcontainers/device/drivers
-	device_manager=$runtimedir/virtcontainers/device/manager
+	device_api=$runtimedir/pkg/device/api
+	device_config=$runtimedir/pkg/device/config
+	device_drivers=$runtimedir/pkg/device/drivers
+	device_manager=$runtimedir/pkg/device/manager
 	rc_pkg_dir=$runtimedir/pkg/resourcecontrol/
 	utils_pkg_dir=$runtimedir/virtcontainers/utils
 

snap/snapcraft.yaml

@@ -74,7 +74,7 @@ parts:
       rustup toolchain install ${version}
       rustup default ${version}
       if [ "${arch}" == "ppc64le" ] || [ "${arch}" == "s390x" ] ; then
-        [ ${arch} == "ppc64le"] && arch="powerpc64le"
+        [ "${arch}" == "ppc64le" ] && arch="powerpc64le"
         rustup target add ${arch}-unknown-linux-gnu
       else
         rustup target add ${arch}-unknown-linux-musl

src/agent/Makefile

@@ -110,14 +110,14 @@ default: $(TARGET) show-header
 $(TARGET): $(GENERATED_CODE) $(TARGET_PATH)
 
 $(TARGET_PATH): show-summary
-	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="$(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
 
 $(GENERATED_FILES): %: %.in
 	@sed $(foreach r,$(GENERATED_REPLACEMENTS),-e 's|@$r@|$($r)|g') "$<" > "$@"
 
 ##TARGET optimize: optimized  build
 optimize: show-summary show-header
-	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES)
+	@RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES)
 
 ##TARGET install: install agent
 install: install-services

src/agent/src/netlink.rs

@@ -523,7 +523,7 @@ impl Handle {
             .as_ref()
             .map(|to| to.address.as_str()) // Extract address field
             .and_then(|addr| if addr.is_empty() { None } else { Some(addr) }) // Make sure it's not empty
-            .ok_or(anyhow!(nix::Error::EINVAL))?;
+            .ok_or_else(|| anyhow!(nix::Error::EINVAL))?;
 
         let ip = IpAddr::from_str(ip_address)
             .map_err(|e| anyhow!("Failed to parse IP {}: {:?}", ip_address, e))?;
@@ -612,7 +612,7 @@ fn parse_mac_address(addr: &str) -> Result<[u8; 6]> {
 
     // Parse single Mac address block
     let mut parse_next = || -> Result<u8> {
-        let v = u8::from_str_radix(split.next().ok_or(anyhow!(nix::Error::EINVAL))?, 16)?;
+        let v = u8::from_str_radix(split.next().ok_or_else(|| anyhow!(nix::Error::EINVAL))?, 16)?;
         Ok(v)
     };
 

src/agent/src/random.rs

@@ -82,7 +82,7 @@ mod tests {
         if nix::unistd::Uid::effective().is_root() {
             assert!(ret.is_ok());
         } else {
-            assert!(!ret.is_ok());
+            assert!(ret.is_err());
         }
     }
 
@@ -90,6 +90,6 @@ mod tests {
     fn test_reseed_rng_zero_data() {
         let seed = [];
         let ret = reseed_rng(&seed);
-        assert!(!ret.is_ok());
+        assert!(ret.is_err());
     }
 }

src/runtime/Makefile

@@ -167,6 +167,11 @@ DEFDISABLEGUESTEMPTYDIR := false
 DEFAULTEXPFEATURES := []
 
 DEFDISABLESELINUX := false
+#Default SeccomSandbox param
+#The same default policy is used by libvirt
+#More explanation on https://lists.gnu.org/archive/html/qemu-devel/2017-02/msg03348.html
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+DEFSECCOMPSANDBOXPARAM := on,obsolete=deny,spawn=deny,resourcecontrol=deny
 
 #Default entropy source
 DEFENTROPYSOURCE := /dev/urandom
@@ -459,6 +464,7 @@ USER_VARS += DEFVIRTIOFSCACHE
 USER_VARS += DEFVIRTIOFSEXTRAARGS
 USER_VARS += DEFENABLEANNOTATIONS
 USER_VARS += DEFENABLEIOTHREADS
+USER_VARS += DEFSECCOMPSANDBOXPARAM
 USER_VARS += DEFENABLEVHOSTUSERSTORE
 USER_VARS += DEFVHOSTUSERSTOREPATH
 USER_VARS += DEFVALIDVHOSTUSERSTOREPATHS

src/runtime/config/configuration-qemu.toml.in

@@ -76,6 +76,14 @@ firmware_volume = "@FIRMWAREVOLUMEPATH@"
 # For example, `machine_accelerators = "nosmm,nosmbus,nosata,nopit,static-prt,nofw"`
 machine_accelerators="@MACHINEACCELERATORS@"
 
+# Qemu seccomp sandbox feature
+# comma-separated list of seccomp sandbox features to control the syscall access.
+# For example, `seccompsandbox= "on,obsolete=deny,spawn=deny,resourcecontrol=deny"`
+# Note: "elevateprivileges=deny" doesn't work with daemonize option, so it's removed from the seccomp sandbox
+# Another note: enabling this feature may reduce performance, you may enable
+# /proc/sys/net/core/bpf_jit_enable to reduce the impact. see https://man7.org/linux/man-pages/man8/bpfc.8.html
+#seccompsandbox="@DEFSECCOMPSANDBOXPARAM@"
+
 # CPU features
 # comma-separated list of cpu features to pass to the cpu
 # For example, `cpu_features = "pmu=off,vmx=off"

src/runtime/go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/urfave/cli v1.22.2
 	github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868
 	github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
+	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965
 	go.opentelemetry.io/otel v1.3.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
 	go.opentelemetry.io/otel/sdk v1.3.0

src/runtime/go.sum

@@ -925,6 +925,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs=
 github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 h1:EXE1ZsUqiUWGV5Dw2oTYpXx24ffxj0//yhTB0Ppv+4s=
+gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=

src/runtime/pkg/containerd-shim-v2/create.go

@@ -97,7 +97,7 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest) (*con
 		}
 
 		// create root span
-		rootSpan, newCtx := katatrace.Trace(s.ctx, shimLog, "root span", shimTracingTags)
+		rootSpan, newCtx := katatrace.Trace(s.ctx, shimLog, "rootSpan", shimTracingTags)
 		s.rootCtx = newCtx
 		defer rootSpan.End()
 

src/runtime/pkg/containerd-shim-v2/service.go

@@ -7,6 +7,7 @@ package containerdshim
 
 import (
 	"context"
+	"fmt"
 	"io"
 	"os"
 	sysexec "os/exec"
@@ -85,6 +86,11 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu
 	vci.SetLogger(ctx, shimLog)
 	katautils.SetLogger(ctx, shimLog, shimLog.Logger.Level)
 
+	ns, found := namespaces.Namespace(ctx)
+	if !found {
+		return nil, fmt.Errorf("shim namespace cannot be empty")
+	}
+
 	s := &service{
 		id:         id,
 		pid:        uint32(os.Getpid()),
@@ -93,6 +99,7 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu
 		events:     make(chan interface{}, chSize),
 		ec:         make(chan exit, bufferSize),
 		cancel:     shutdown,
+		namespace:  ns,
 	}
 
 	go s.processExits()
@@ -131,6 +138,9 @@ type service struct {
 
 	id string
 
+	// Namespace from upper container engine
+	namespace string
+
 	mu          sync.Mutex
 	eventSendMu sync.Mutex
 

src/runtime/pkg/containerd-shim-v2/shim_io_binary.go

@@ -0,0 +1,216 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"syscall"
+	"time"
+
+	"golang.org/x/sys/execabs"
+
+	"github.com/hashicorp/go-multierror"
+)
+
+const (
+	binaryIOProcTermTimeout = 12 * time.Second // Give logger process solid 10 seconds for cleanup
+)
+
+var (
+	_ IO = &binaryIO{}
+)
+
+// binaryIO related code is from https://github.com/containerd/containerd/blob/v1.6.6/pkg/process/io.go#L311
+type binaryIO struct {
+	cmd      *execabs.Cmd
+	out, err *pipe
+}
+
+// https://github.com/containerd/containerd/blob/v1.6.6/pkg/process/io.go#L248
+func newBinaryIO(ctx context.Context, ns, id string, uri *url.URL) (bio *binaryIO, err error) {
+	var closers []func() error
+	defer func() {
+		if err == nil {
+			return
+		}
+		result := multierror.Append(err)
+		for _, fn := range closers {
+			result = multierror.Append(result, fn())
+		}
+		err = multierror.Flatten(result)
+	}()
+
+	out, err := newPipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create stdout pipes: %w", err)
+	}
+	closers = append(closers, out.Close)
+
+	serr, err := newPipe()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create stderr pipes: %w", err)
+	}
+	closers = append(closers, serr.Close)
+
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	closers = append(closers, r.Close, w.Close)
+
+	cmd := newBinaryCmd(uri, id, ns)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, out.r, serr.r, w)
+	// don't need to register this with the reaper or wait when
+	// running inside a shim
+	if err := cmd.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start binary process: %w", err)
+	}
+	closers = append(closers, func() error { return cmd.Process.Kill() })
+
+	// close our side of the pipe after start
+	if err := w.Close(); err != nil {
+		return nil, fmt.Errorf("failed to close write pipe after start: %w", err)
+	}
+
+	// wait for the logging binary to be ready
+	b := make([]byte, 1)
+	if _, err := r.Read(b); err != nil && err != io.EOF {
+		return nil, fmt.Errorf("failed to read from logging binary: %w", err)
+	}
+
+	return &binaryIO{
+		cmd: cmd,
+		out: out,
+		err: serr,
+	}, nil
+}
+
+// newBinaryCmd returns a Cmd to be used to start a logging binary.
+// The Cmd is generated from the provided uri, and the container ID and
+// namespace are appended to the Cmd environment.
+func newBinaryCmd(binaryURI *url.URL, id, ns string) *execabs.Cmd {
+	var args []string
+	for k, vs := range binaryURI.Query() {
+		args = append(args, k)
+		if len(vs) > 0 {
+			args = append(args, vs[0])
+		}
+	}
+
+	cmd := execabs.Command(binaryURI.Path, args...)
+
+	cmd.Env = append(cmd.Env,
+		"CONTAINER_ID="+id,
+		"CONTAINER_NAMESPACE="+ns,
+	)
+
+	return cmd
+}
+
+func (bi *binaryIO) Stdin() io.ReadCloser {
+	return nil
+}
+
+func (bi *binaryIO) Stdout() io.Writer {
+	return bi.out.w
+}
+
+func (bi *binaryIO) Stderr() io.Writer {
+	return bi.err.w
+}
+
+func (bi *binaryIO) Close() error {
+	var (
+		result *multierror.Error
+	)
+
+	for _, v := range []*pipe{bi.out, bi.err} {
+		if v != nil {
+			if err := v.Close(); err != nil {
+				result = multierror.Append(result, err)
+			}
+		}
+	}
+
+	if err := bi.cancel(); err != nil {
+		result = multierror.Append(result, err)
+	}
+
+	return result.ErrorOrNil()
+}
+
+func (bi *binaryIO) cancel() error {
+	if bi.cmd == nil || bi.cmd.Process == nil {
+		return nil
+	}
+
+	// Send SIGTERM first, so logger process has a chance to flush and exit properly
+	if err := bi.cmd.Process.Signal(syscall.SIGTERM); err != nil {
+		result := multierror.Append(fmt.Errorf("failed to send SIGTERM: %w", err))
+
+		shimLog.WithError(err).Warn("failed to send SIGTERM signal, killing logging shim")
+
+		if err := bi.cmd.Process.Kill(); err != nil {
+			result = multierror.Append(result, fmt.Errorf("failed to kill process after faulty SIGTERM: %w", err))
+		}
+
+		return result.ErrorOrNil()
+	}
+
+	done := make(chan error, 1)
+	go func() {
+		done <- bi.cmd.Wait()
+	}()
+
+	select {
+	case err := <-done:
+		return err
+	case <-time.After(binaryIOProcTermTimeout):
+		shimLog.Warn("failed to wait for shim logger process to exit, killing")
+
+		err := bi.cmd.Process.Kill()
+		if err != nil {
+			return fmt.Errorf("failed to kill shim logger process: %w", err)
+		}
+
+		return nil
+	}
+}
+
+func newPipe() (*pipe, error) {
+	r, w, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	return &pipe{
+		r: r,
+		w: w,
+	}, nil
+}
+
+type pipe struct {
+	r *os.File
+	w *os.File
+}
+
+// https://github.com/containerd/containerd/blob/v1.6.6/vendor/github.com/containerd/go-runc/io.go#L71
+func (p *pipe) Close() error {
+	var result *multierror.Error
+
+	if err := p.w.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close write pipe: %w", err))
+	}
+
+	if err := p.r.Close(); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close read pipe: %w", err))
+	}
+
+	return multierror.Prefix(result.ErrorOrNil(), "pipe:")
+}

src/runtime/pkg/containerd-shim-v2/shim_io_file.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"io"
+	"net/url"
+	"os"
+	"path/filepath"
+
+	cioutil "github.com/containerd/containerd/pkg/ioutil"
+)
+
+var (
+	_ IO = &fileIO{}
+)
+
+// fileIO only support write both stdout/stderr to the same file
+type fileIO struct {
+	outw io.WriteCloser
+	errw io.WriteCloser
+	path string
+}
+
+// openLogFile opens/creates a container log file with its directory.
+func openLogFile(path string) (*os.File, error) {
+	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+		return nil, err
+	}
+	return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640)
+}
+
+func newFileIO(ctx context.Context, stdio *stdio, uri *url.URL) (*fileIO, error) {
+	var outw, errw, f io.WriteCloser
+	var err error
+
+	logFile := uri.Path
+	if f, err = openLogFile(logFile); err != nil {
+		return nil, err
+	}
+
+	if stdio.Stdout != "" {
+		outw = cioutil.NewSerialWriteCloser(f)
+	}
+
+	if !stdio.Console && stdio.Stderr != "" {
+		errw = cioutil.NewSerialWriteCloser(f)
+	}
+
+	return &fileIO{
+		path: logFile,
+		outw: outw,
+		errw: errw,
+	}, nil
+}
+
+func (fi *fileIO) Close() error {
+	if fi.outw != nil {
+		return wc(fi.outw)
+	} else if fi.errw != nil {
+		return wc(fi.errw)
+	}
+	return nil
+}
+
+func (fi *fileIO) Stdin() io.ReadCloser {
+	return nil
+}
+
+func (fi *fileIO) Stdout() io.Writer {
+	return fi.outw
+}
+
+func (fi *fileIO) Stderr() io.Writer {
+	return fi.errw
+}

src/runtime/pkg/containerd-shim-v2/shim_io_pipe.go

@@ -0,0 +1,95 @@
+// Copyright (c) 2022 Ant Group
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+
+package containerdshim
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"syscall"
+
+	"github.com/containerd/fifo"
+	"github.com/hashicorp/go-multierror"
+)
+
+var (
+	_ IO = &pipeIO{}
+)
+
+type pipeIO struct {
+	in   io.ReadCloser
+	outw io.WriteCloser
+	errw io.WriteCloser
+}
+
+func newPipeIO(ctx context.Context, stdio *stdio) (*pipeIO, error) {
+	var in io.ReadCloser
+	var outw io.WriteCloser
+	var errw io.WriteCloser
+	var err error
+
+	if stdio.Stdin != "" {
+		in, err = fifo.OpenFifo(ctx, stdio.Stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if stdio.Stdout != "" {
+		outw, err = fifo.OpenFifo(ctx, stdio.Stdout, syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	if !stdio.Console && stdio.Stderr != "" {
+		errw, err = fifo.OpenFifo(ctx, stdio.Stderr, syscall.O_RDWR, 0)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	pipeIO := &pipeIO{
+		in:   in,
+		outw: outw,
+		errw: errw,
+	}
+
+	return pipeIO, nil
+}
+
+func (pi *pipeIO) Stdin() io.ReadCloser {
+	return pi.in
+}
+
+func (pi *pipeIO) Stdout() io.Writer {
+	return pi.outw
+}
+
+func (pi *pipeIO) Stderr() io.Writer {
+	return pi.errw
+}
+
+func (pi *pipeIO) Close() error {
+	var result *multierror.Error
+
+	if pi.in != nil {
+		if err := pi.in.Close(); err != nil {
+			result = multierror.Append(result, fmt.Errorf("failed to close stdin: %w", err))
+		}
+		pi.in = nil
+	}
+
+	if err := wc(pi.outw); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close stdout: %w", err))
+	}
+
+	if err := wc(pi.errw); err != nil {
+		result = multierror.Append(result, fmt.Errorf("failed to close stderr: %w", err))
+	}
+
+	return result.ErrorOrNil()
+}

src/runtime/pkg/containerd-shim-v2/start.go

@@ -8,6 +8,7 @@ package containerdshim
 import (
 	"context"
 	"fmt"
+
 	"github.com/sirupsen/logrus"
 
 	"github.com/containerd/containerd/api/types/task"
@@ -75,7 +76,7 @@ func startContainer(ctx context.Context, s *service, c *container) (retErr error
 	c.stdinPipe = stdin
 
 	if c.stdin != "" || c.stdout != "" || c.stderr != "" {
-		tty, err := newTtyIO(ctx, c.stdin, c.stdout, c.stderr, c.terminal)
+		tty, err := newTtyIO(ctx, s.namespace, c.id, c.stdin, c.stdout, c.stderr, c.terminal)
 		if err != nil {
 			return err
 		}
@@ -141,7 +142,7 @@ func startExec(ctx context.Context, s *service, containerID, execID string) (e *
 
 	execs.stdinPipe = stdin
 
-	tty, err := newTtyIO(ctx, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal)
+	tty, err := newTtyIO(ctx, s.namespace, execs.id, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal)
 	if err != nil {
 		return nil, err
 	}

src/runtime/pkg/containerd-shim-v2/stream.go

@@ -7,16 +7,22 @@ package containerdshim
 
 import (
 	"context"
+	"fmt"
 	"io"
+	"net/url"
 	"sync"
-	"syscall"
 
-	"github.com/containerd/fifo"
 	"github.com/sirupsen/logrus"
 )
 
-// The buffer size used to specify the buffer for IO streams copy
-const bufSize = 32 << 10
+const (
+	// The buffer size used to specify the buffer for IO streams copy
+	bufSize = 32 << 10
+
+	shimLogPluginBinary = "binary"
+	shimLogPluginFifo   = "fifo"
+	shimLogPluginFile   = "file"
+)
 
 var (
 	bufPool = sync.Pool{
@@ -27,76 +33,84 @@ var (
 	}
 )
 
+type stdio struct {
+	Stdin   string
+	Stdout  string
+	Stderr  string
+	Console bool
+}
+type IO interface {
+	io.Closer
+	Stdin() io.ReadCloser
+	Stdout() io.Writer
+	Stderr() io.Writer
+}
+
 type ttyIO struct {
-	Stdin  io.ReadCloser
-	Stdout io.Writer
-	Stderr io.Writer
+	io  IO
+	raw *stdio
 }
 
 func (tty *ttyIO) close() {
-
-	if tty.Stdin != nil {
-		tty.Stdin.Close()
-		tty.Stdin = nil
-	}
-	cf := func(w io.Writer) {
-		if w == nil {
-			return
-		}
-		if c, ok := w.(io.WriteCloser); ok {
-			c.Close()
-		}
-	}
-	cf(tty.Stdout)
-	cf(tty.Stderr)
+	tty.io.Close()
 }
 
-func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (*ttyIO, error) {
-	var in io.ReadCloser
-	var outw io.Writer
-	var errw io.Writer
+// newTtyIO creates a new ttyIO struct.
+// ns(namespace)/id(container ID) are used for containerd binary IO.
+// containerd will pass the ns/id as ENV to the binary log driver,
+// and the binary log driver will use ns/id to get the log options config file.
+// for example nerdctl: https://github.com/containerd/nerdctl/blob/v0.21.0/pkg/logging/logging.go#L102
+func newTtyIO(ctx context.Context, ns, id, stdin, stdout, stderr string, console bool) (*ttyIO, error) {
 	var err error
+	var io IO
 
-	if stdin != "" {
-		in, err = fifo.OpenFifo(ctx, stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0)
-		if err != nil {
-			return nil, err
-		}
+	raw := &stdio{
+		Stdin:   stdin,
+		Stdout:  stdout,
+		Stderr:  stderr,
+		Console: console,
 	}
 
-	if stdout != "" {
-		outw, err = fifo.OpenFifo(ctx, stdout, syscall.O_RDWR, 0)
-		if err != nil {
-			return nil, err
-		}
+	uri, err := url.Parse(stdout)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse stdout uri: %w", err)
 	}
 
-	if !console && stderr != "" {
-		errw, err = fifo.OpenFifo(ctx, stderr, syscall.O_RDWR, 0)
-		if err != nil {
-			return nil, err
-		}
+	if uri.Scheme == "" {
+		uri.Scheme = "fifo"
 	}
 
-	ttyIO := &ttyIO{
-		Stdin:  in,
-		Stdout: outw,
-		Stderr: errw,
+	switch uri.Scheme {
+	case shimLogPluginFifo:
+		io, err = newPipeIO(ctx, raw)
+	case shimLogPluginBinary:
+		io, err = newBinaryIO(ctx, ns, id, uri)
+	case shimLogPluginFile:
+		io, err = newFileIO(ctx, raw, uri)
+	default:
+		return nil, fmt.Errorf("unknown STDIO scheme %s", uri.Scheme)
 	}
 
-	return ttyIO, nil
+	if err != nil {
+		return nil, fmt.Errorf("failed to creat io stream: %w", err)
+	}
+
+	return &ttyIO{
+		io:  io,
+		raw: raw,
+	}, nil
 }
 
 func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) {
 	var wg sync.WaitGroup
 
-	if tty.Stdin != nil {
+	if tty.io.Stdin() != nil {
 		wg.Add(1)
 		go func() {
 			shimLog.Debug("stdin io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
-			io.CopyBuffer(stdinPipe, tty.Stdin, *p)
+			io.CopyBuffer(stdinPipe, tty.io.Stdin(), *p)
 			// notify that we can close process's io safely.
 			close(stdinCloser)
 			wg.Done()
@@ -104,30 +118,30 @@ func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO
 		}()
 	}
 
-	if tty.Stdout != nil {
+	if tty.io.Stdout() != nil {
 		wg.Add(1)
 
 		go func() {
 			shimLog.Debug("stdout io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
-			io.CopyBuffer(tty.Stdout, stdoutPipe, *p)
+			io.CopyBuffer(tty.io.Stdout(), stdoutPipe, *p)
 			wg.Done()
-			if tty.Stdin != nil {
+			if tty.io.Stdin() != nil {
 				// close stdin to make the other routine stop
-				tty.Stdin.Close()
+				tty.io.Stdin().Close()
 			}
 			shimLog.Debug("stdout io stream copy exited")
 		}()
 	}
 
-	if tty.Stderr != nil && stderrPipe != nil {
+	if tty.io.Stderr() != nil && stderrPipe != nil {
 		wg.Add(1)
 		go func() {
 			shimLog.Debug("stderr io stream copy started")
 			p := bufPool.Get().(*[]byte)
 			defer bufPool.Put(p)
-			io.CopyBuffer(tty.Stderr, stderrPipe, *p)
+			io.CopyBuffer(tty.io.Stderr(), stderrPipe, *p)
 			wg.Done()
 			shimLog.Debug("stderr io stream copy exited")
 		}()
@@ -138,3 +152,10 @@ func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO
 	close(exitch)
 	shimLog.Debug("all io stream copy goroutines exited")
 }
+
+func wc(w io.WriteCloser) error {
+	if w == nil {
+		return nil
+	}
+	return w.Close()
+}

src/runtime/pkg/containerd-shim-v2/stream_test.go

@@ -7,7 +7,6 @@ package containerdshim
 
 import (
 	"context"
-	"github.com/sirupsen/logrus"
 	"io"
 	"os"
 	"path/filepath"
@@ -15,6 +14,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/sirupsen/logrus"
+
 	"github.com/containerd/fifo"
 	"github.com/stretchr/testify/assert"
 )
@@ -45,7 +46,7 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 	defer outr.Close()
 	errr = createReadFifo(stderr)
 	defer errr.Close()
-	tty, err = newTtyIO(ctx, "", stdout, stderr, false)
+	tty, err = newTtyIO(ctx, "", "", "", stdout, stderr, false)
 	assert.NoError(err)
 	defer tty.close()
 
@@ -72,9 +73,9 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 		}
 	}
 
-	checkFifoWrite(tty.Stdout)
+	checkFifoWrite(tty.io.Stdout())
 	checkFifoRead(outr)
-	checkFifoWrite(tty.Stderr)
+	checkFifoWrite(tty.io.Stderr())
 	checkFifoRead(errr)
 
 	err = outr.Close()
@@ -84,8 +85,8 @@ func TestNewTtyIOFifoReopen(t *testing.T) {
 
 	// Make sure that writing to tty fifo will not get `EPIPE`
 	// when the read side is closed
-	checkFifoWrite(tty.Stdout)
-	checkFifoWrite(tty.Stderr)
+	checkFifoWrite(tty.io.Stdout())
+	checkFifoWrite(tty.io.Stderr())
 
 	// Reopen the fifo
 	outr = createReadFifo(stdout)
@@ -171,7 +172,7 @@ func TestIoCopy(t *testing.T) {
 			defer srcInW.Close()
 		}
 
-		tty, err := newTtyIO(ctx, srcStdinPath, dstStdoutPath, dstStderrPath, false)
+		tty, err := newTtyIO(ctx, "", "", srcStdinPath, dstStdoutPath, dstStderrPath, false)
 		assert.NoError(err)
 		defer tty.close()
 

src/runtime/pkg/device/api/interface.go

@@ -9,8 +9,7 @@ package api
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/sirupsen/logrus"
 )
 
@@ -78,10 +77,10 @@ type Device interface {
 	Dereference() uint
 
 	// Save converts Device to DeviceState
-	Save() persistapi.DeviceState
+	Save() config.DeviceState
 
 	// Load loads DeviceState and converts it to specific device
-	Load(persistapi.DeviceState)
+	Load(config.DeviceState)
 }
 
 // DeviceManager can be used to create a new device, this can be used as single
@@ -94,5 +93,5 @@ type DeviceManager interface {
 	IsDeviceAttached(string) bool
 	GetDeviceByID(string) Device
 	GetAllDevices() []Device
-	LoadDevices([]persistapi.DeviceState)
+	LoadDevices([]config.DeviceState)
 }

src/runtime/pkg/device/api/mockDeviceReceiver.go

@@ -8,7 +8,7 @@ package api
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 // MockDeviceReceiver is a fake DeviceReceiver API implementation only used for test

src/runtime/pkg/device/config/config.go

@@ -441,3 +441,45 @@ func getVhostUserDevName(dirname string, majorNum, minorNum uint32) (string, err
 	return "", fmt.Errorf("Required device node (%d:%d) doesn't exist under directory %s",
 		majorNum, minorNum, dirname)
 }
+
+// DeviceState is a structure which represents host devices
+// plugged to a hypervisor, one Device can be shared among containers in POD
+// Refs: virtcontainers/device/drivers/generic.go:GenericDevice
+type DeviceState struct {
+	// DriverOptions is specific options for each device driver
+	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
+	DriverOptions map[string]string
+
+	// VhostUserDeviceAttrs is specific for vhost-user device driver
+	VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"`
+
+	// BlockDrive is specific for block device driver
+	BlockDrive *BlockDrive `json:",omitempty"`
+
+	ID string
+
+	// Type is used to specify driver type
+	// Refs: virtcontainers/device/config/config.go:DeviceType
+	Type string
+
+	// Type of device: c, b, u or p
+	// c , u - character(unbuffered)
+	// p - FIFO
+	// b - block(buffered) special file
+	// More info in mknod(1).
+	DevType string
+
+	// VFIODev is specific VFIO device driver
+	VFIODevs []*VFIODev `json:",omitempty"`
+
+	RefCount    uint
+	AttachCount uint
+
+	// Major, minor numbers for device.
+	Major int64
+	Minor int64
+
+	// ColdPlug specifies whether the device must be cold plugged (true)
+	// or hot plugged (false).
+	ColdPlug bool
+}

src/runtime/pkg/device/drivers/block.go

@@ -10,9 +10,8 @@ import (
 	"context"
 	"path/filepath"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -159,51 +158,21 @@ func (device *BlockDevice) GetDeviceInfo() interface{} {
 }
 
 // Save converts Device to DeviceState
-func (device *BlockDevice) Save() persistapi.DeviceState {
+func (device *BlockDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
 
-	drive := device.BlockDrive
-	if drive != nil {
-		ds.BlockDrive = &persistapi.BlockDrive{
-			File:     drive.File,
-			Format:   drive.Format,
-			ID:       drive.ID,
-			Index:    drive.Index,
-			MmioAddr: drive.MmioAddr,
-			PCIPath:  drive.PCIPath,
-			SCSIAddr: drive.SCSIAddr,
-			NvdimmID: drive.NvdimmID,
-			VirtPath: drive.VirtPath,
-			DevNo:    drive.DevNo,
-			Pmem:     drive.Pmem,
-		}
-	}
+	ds.BlockDrive = device.BlockDrive
+
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *BlockDevice) Load(ds persistapi.DeviceState) {
+func (device *BlockDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
-	bd := ds.BlockDrive
-	if bd == nil {
-		return
-	}
-	device.BlockDrive = &config.BlockDrive{
-		File:     bd.File,
-		Format:   bd.Format,
-		ID:       bd.ID,
-		Index:    bd.Index,
-		MmioAddr: bd.MmioAddr,
-		PCIPath:  bd.PCIPath,
-		SCSIAddr: bd.SCSIAddr,
-		NvdimmID: bd.NvdimmID,
-		VirtPath: bd.VirtPath,
-		DevNo:    bd.DevNo,
-		Pmem:     bd.Pmem,
-	}
+	device.BlockDrive = ds.BlockDrive
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/drivers/generic.go

@@ -10,9 +10,8 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 // GenericDevice refers to a device that is neither a VFIO device, block device or VhostUserDevice.
@@ -128,8 +127,8 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error)
 }
 
 // Save converts Device to DeviceState
-func (device *GenericDevice) Save() persistapi.DeviceState {
-	dss := persistapi.DeviceState{
+func (device *GenericDevice) Save() config.DeviceState {
+	dss := config.DeviceState{
 		ID:          device.ID,
 		Type:        string(device.DeviceType()),
 		RefCount:    device.RefCount,
@@ -148,7 +147,7 @@ func (device *GenericDevice) Save() persistapi.DeviceState {
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *GenericDevice) Load(ds persistapi.DeviceState) {
+func (device *GenericDevice) Load(ds config.DeviceState) {
 	device.ID = ds.ID
 	device.RefCount = ds.RefCount
 	device.AttachCount = ds.AttachCount

src/runtime/pkg/device/drivers/generic_test.go

@@ -8,7 +8,7 @@ package drivers
 import (
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/pkg/device/drivers/utils.go

@@ -12,8 +12,8 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/sirupsen/logrus"
 )
 

src/runtime/pkg/device/drivers/vfio.go

@@ -16,9 +16,8 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -174,26 +173,21 @@ func (device *VFIODevice) GetDeviceInfo() interface{} {
 }
 
 // Save converts Device to DeviceState
-func (device *VFIODevice) Save() persistapi.DeviceState {
+func (device *VFIODevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
 
 	devs := device.VfioDevs
 	for _, dev := range devs {
 		if dev != nil {
-			ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{
-				ID:       dev.ID,
-				Type:     uint32(dev.Type),
-				BDF:      dev.BDF,
-				SysfsDev: dev.SysfsDev,
-			})
+			ds.VFIODevs = append(ds.VFIODevs, dev)
 		}
 	}
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VFIODevice) Load(ds persistapi.DeviceState) {
+func (device *VFIODevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 

src/runtime/pkg/device/drivers/vfio_test.go

@@ -9,7 +9,7 @@ package drivers
 import (
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/pkg/device/drivers/vhost_user_blk.go

@@ -9,9 +9,8 @@ package drivers
 import (
 	"context"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
 )
@@ -156,40 +155,19 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} {
 }
 
 // Save converts Device to DeviceState
-func (device *VhostUserBlkDevice) Save() persistapi.DeviceState {
+func (device *VhostUserBlkDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
+	ds.VhostUserDev = device.VhostUserDeviceAttrs
 
-	vAttr := device.VhostUserDeviceAttrs
-	if vAttr != nil {
-		ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-			DevID:      vAttr.DevID,
-			SocketPath: vAttr.SocketPath,
-			Type:       string(vAttr.Type),
-			PCIPath:    vAttr.PCIPath,
-			Index:      vAttr.Index,
-		}
-	}
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VhostUserBlkDevice) Load(ds persistapi.DeviceState) {
+func (device *VhostUserBlkDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
-
-	dev := ds.VhostUserDev
-	if dev == nil {
-		return
-	}
-
-	device.VhostUserDeviceAttrs = &config.VhostUserDeviceAttrs{
-		DevID:      dev.DevID,
-		SocketPath: dev.SocketPath,
-		Type:       config.DeviceType(dev.Type),
-		PCIPath:    dev.PCIPath,
-		Index:      dev.Index,
-	}
+	device.VhostUserDeviceAttrs = ds.VhostUserDev
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/drivers/vhost_user_fs.go

@@ -9,8 +9,8 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 

src/runtime/pkg/device/drivers/vhost_user_net.go

@@ -10,16 +10,15 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
 // VhostUserNetDevice is a network vhost-user based device
 type VhostUserNetDevice struct {
 	*GenericDevice
-	config.VhostUserDeviceAttrs
+	*config.VhostUserDeviceAttrs
 }
 
 //
@@ -71,38 +70,25 @@ func (device *VhostUserNetDevice) DeviceType() config.DeviceType {
 // GetDeviceInfo returns device information used for creating
 func (device *VhostUserNetDevice) GetDeviceInfo() interface{} {
 	device.Type = device.DeviceType()
-	return &device.VhostUserDeviceAttrs
+	return device.VhostUserDeviceAttrs
 }
 
 // Save converts Device to DeviceState
-func (device *VhostUserNetDevice) Save() persistapi.DeviceState {
+func (device *VhostUserNetDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
-	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-		DevID:      device.DevID,
-		SocketPath: device.SocketPath,
-		Type:       string(device.Type),
-		MacAddress: device.MacAddress,
-	}
+
+	ds.VhostUserDev = device.VhostUserDeviceAttrs
+
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VhostUserNetDevice) Load(ds persistapi.DeviceState) {
+func (device *VhostUserNetDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
-	dev := ds.VhostUserDev
-	if dev == nil {
-		return
-	}
-
-	device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{
-		DevID:      dev.DevID,
-		SocketPath: dev.SocketPath,
-		Type:       config.DeviceType(dev.Type),
-		MacAddress: dev.MacAddress,
-	}
+	device.VhostUserDeviceAttrs = ds.VhostUserDev
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/drivers/vhost_user_scsi.go

@@ -10,16 +10,15 @@ import (
 	"context"
 	"encoding/hex"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
 // VhostUserSCSIDevice is a SCSI vhost-user based device
 type VhostUserSCSIDevice struct {
 	*GenericDevice
-	config.VhostUserDeviceAttrs
+	*config.VhostUserDeviceAttrs
 }
 
 //
@@ -71,38 +70,24 @@ func (device *VhostUserSCSIDevice) DeviceType() config.DeviceType {
 // GetDeviceInfo returns device information used for creating
 func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} {
 	device.Type = device.DeviceType()
-	return &device.VhostUserDeviceAttrs
+	return device.VhostUserDeviceAttrs
 }
 
 // Save converts Device to DeviceState
-func (device *VhostUserSCSIDevice) Save() persistapi.DeviceState {
+func (device *VhostUserSCSIDevice) Save() config.DeviceState {
 	ds := device.GenericDevice.Save()
 	ds.Type = string(device.DeviceType())
-	ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{
-		DevID:      device.DevID,
-		SocketPath: device.SocketPath,
-		Type:       string(device.Type),
-		MacAddress: device.MacAddress,
-	}
+	ds.VhostUserDev = device.VhostUserDeviceAttrs
+
 	return ds
 }
 
 // Load loads DeviceState and converts it to specific device
-func (device *VhostUserSCSIDevice) Load(ds persistapi.DeviceState) {
+func (device *VhostUserSCSIDevice) Load(ds config.DeviceState) {
 	device.GenericDevice = &GenericDevice{}
 	device.GenericDevice.Load(ds)
 
-	dev := ds.VhostUserDev
-	if dev == nil {
-		return
-	}
-
-	device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{
-		DevID:      dev.DevID,
-		SocketPath: dev.SocketPath,
-		Type:       config.DeviceType(dev.Type),
-		MacAddress: dev.MacAddress,
-	}
+	device.VhostUserDeviceAttrs = ds.VhostUserDev
 }
 
 // It should implement GetAttachCount() and DeviceID() as api.Device implementation

src/runtime/pkg/device/manager/manager.go

@@ -14,10 +14,9 @@ import (
 
 	"github.com/sirupsen/logrus"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
-	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
 
@@ -242,7 +241,7 @@ func (dm *deviceManager) IsDeviceAttached(id string) bool {
 }
 
 // LoadDevices load devices from persist state
-func (dm *deviceManager) LoadDevices(devStates []persistapi.DeviceState) {
+func (dm *deviceManager) LoadDevices(devStates []config.DeviceState) {
 	dm.Lock()
 	defer dm.Unlock()
 

src/runtime/pkg/device/manager/manager_linux_test.go

@@ -13,10 +13,10 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
 	"github.com/stretchr/testify/assert"
 
 	"golang.org/x/sys/unix"

src/runtime/pkg/device/manager/manager_test.go

@@ -13,9 +13,9 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/pkg/device/manager/utils.go

@@ -10,7 +10,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 const (

src/runtime/pkg/device/manager/utils_test.go

@@ -9,7 +9,7 @@ package manager
 import (
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/pkg/govmm/qemu/qemu.go

@@ -15,6 +15,7 @@ package qemu
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"log"
 	"os"
@@ -23,8 +24,6 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
-
-	"context"
 )
 
 // Machine describes the machine type qemu will emulate.

src/runtime/pkg/katautils/config.go

@@ -16,12 +16,12 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/oci"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
@@ -95,6 +95,7 @@ type hypervisor struct {
 	FileBackedMemRootDir           string   `toml:"file_mem_backend"`
 	GuestHookPath                  string   `toml:"guest_hook_path"`
 	GuestMemoryDumpPath            string   `toml:"guest_memory_dump_path"`
+	SeccompSandbox                 string   `toml:"seccompsandbox"`
 	HypervisorPathList             []string `toml:"valid_hypervisor_paths"`
 	JailerPathList                 []string `toml:"valid_jailer_paths"`
 	CtlPathList                    []string `toml:"valid_ctlpaths"`
@@ -767,6 +768,7 @@ func newQemuHypervisorConfig(h hypervisor) (vc.HypervisorConfig, error) {
 		EnableVhostUserStore:    h.EnableVhostUserStore,
 		VhostUserStorePath:      h.vhostUserStorePath(),
 		VhostUserStorePathList:  h.VhostUserStorePathList,
+		SeccompSandbox:          h.SeccompSandbox,
 		GuestHookPath:           h.guestHookPath(),
 		RxRateLimiterMaxRate:    rxRateLimiterMaxRate,
 		TxRateLimiterMaxRate:    txRateLimiterMaxRate,

src/runtime/pkg/oci/utils.go

@@ -27,7 +27,7 @@ import (
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	dockershimAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim"

src/runtime/pkg/oci/utils_test.go

@@ -21,8 +21,8 @@ import (
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/sys/unix"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	dockerAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"

src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/read_closer.go

@@ -0,0 +1,57 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import "io"
+
+// writeCloseInformer wraps a reader with a close function.
+type wrapReadCloser struct {
+	reader *io.PipeReader
+	writer *io.PipeWriter
+}
+
+// NewWrapReadCloser creates a wrapReadCloser from a reader.
+// NOTE(random-liu): To avoid goroutine leakage, the reader passed in
+// must be eventually closed by the caller.
+func NewWrapReadCloser(r io.Reader) io.ReadCloser {
+	pr, pw := io.Pipe()
+	go func() {
+		_, _ = io.Copy(pw, r)
+		pr.Close()
+		pw.Close()
+	}()
+	return &wrapReadCloser{
+		reader: pr,
+		writer: pw,
+	}
+}
+
+// Read reads up to len(p) bytes into p.
+func (w *wrapReadCloser) Read(p []byte) (int, error) {
+	n, err := w.reader.Read(p)
+	if err == io.ErrClosedPipe {
+		return n, io.EOF
+	}
+	return n, err
+}
+
+// Close closes read closer.
+func (w *wrapReadCloser) Close() error {
+	w.reader.Close()
+	w.writer.Close()
+	return nil
+}

src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/write_closer.go

@@ -0,0 +1,102 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"io"
+	"sync"
+)
+
+// writeCloseInformer wraps passed in write closer with a close channel.
+// Caller could wait on the close channel for the write closer to be
+// closed.
+type writeCloseInformer struct {
+	close chan struct{}
+	wc    io.WriteCloser
+}
+
+// NewWriteCloseInformer creates the writeCloseInformer from a write closer.
+func NewWriteCloseInformer(wc io.WriteCloser) (io.WriteCloser, <-chan struct{}) {
+	close := make(chan struct{})
+	return &writeCloseInformer{
+		close: close,
+		wc:    wc,
+	}, close
+}
+
+// Write passes through the data into the internal write closer.
+func (w *writeCloseInformer) Write(p []byte) (int, error) {
+	return w.wc.Write(p)
+}
+
+// Close closes the internal write closer and inform the close channel.
+func (w *writeCloseInformer) Close() error {
+	err := w.wc.Close()
+	close(w.close)
+	return err
+}
+
+// nopWriteCloser wraps passed in writer with a nop close function.
+type nopWriteCloser struct {
+	w io.Writer
+}
+
+// NewNopWriteCloser creates the nopWriteCloser from a writer.
+func NewNopWriteCloser(w io.Writer) io.WriteCloser {
+	return &nopWriteCloser{w: w}
+}
+
+// Write passes through the data into the internal writer.
+func (n *nopWriteCloser) Write(p []byte) (int, error) {
+	return n.w.Write(p)
+}
+
+// Close is a nop close function.
+func (n *nopWriteCloser) Close() error {
+	return nil
+}
+
+// serialWriteCloser wraps a write closer and makes sure all writes
+// are done in serial.
+// Parallel write won't intersect with each other. Use case:
+// 1) Pipe: Write content longer than PIPE_BUF.
+//    See http://man7.org/linux/man-pages/man7/pipe.7.html
+// 2) <3.14 Linux Kernel: write is not atomic
+//    See http://man7.org/linux/man-pages/man2/write.2.html
+type serialWriteCloser struct {
+	mu sync.Mutex
+	wc io.WriteCloser
+}
+
+// NewSerialWriteCloser creates a SerialWriteCloser from a write closer.
+func NewSerialWriteCloser(wc io.WriteCloser) io.WriteCloser {
+	return &serialWriteCloser{wc: wc}
+}
+
+// Write writes a group of byte arrays in order atomically.
+func (s *serialWriteCloser) Write(data []byte) (int, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.wc.Write(data)
+}
+
+// Close closes the write closer.
+func (s *serialWriteCloser) Close() error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.wc.Close()
+}

src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/writer_group.go

@@ -0,0 +1,105 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ioutil
+
+import (
+	"errors"
+	"io"
+	"sync"
+)
+
+// WriterGroup is a group of writers. Writer could be dynamically
+// added and removed.
+type WriterGroup struct {
+	mu      sync.Mutex
+	writers map[string]io.WriteCloser
+	closed  bool
+}
+
+var _ io.Writer = &WriterGroup{}
+
+// NewWriterGroup creates an empty writer group.
+func NewWriterGroup() *WriterGroup {
+	return &WriterGroup{
+		writers: make(map[string]io.WriteCloser),
+	}
+}
+
+// Add adds a writer into the group. The writer will be closed
+// if the writer group is closed.
+func (g *WriterGroup) Add(key string, w io.WriteCloser) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	if g.closed {
+		w.Close()
+		return
+	}
+	g.writers[key] = w
+}
+
+// Get gets a writer from the group, returns nil if the writer
+// doesn't exist.
+func (g *WriterGroup) Get(key string) io.WriteCloser {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	return g.writers[key]
+}
+
+// Remove removes a writer from the group.
+func (g *WriterGroup) Remove(key string) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	w, ok := g.writers[key]
+	if !ok {
+		return
+	}
+	w.Close()
+	delete(g.writers, key)
+}
+
+// Write writes data into each writer. If a writer returns error,
+// it will be closed and removed from the writer group. It returns
+// error if writer group is empty.
+func (g *WriterGroup) Write(p []byte) (int, error) {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	for k, w := range g.writers {
+		n, err := w.Write(p)
+		if err == nil && len(p) == n {
+			continue
+		}
+		// The writer is closed or in bad state, remove it.
+		w.Close()
+		delete(g.writers, k)
+	}
+	if len(g.writers) == 0 {
+		return 0, errors.New("writer group is empty")
+	}
+	return len(p), nil
+}
+
+// Close closes the writer group. Write will return error after
+// closed.
+func (g *WriterGroup) Close() {
+	g.mu.Lock()
+	defer g.mu.Unlock()
+	for _, w := range g.writers {
+		w.Close()
+	}
+	g.writers = nil
+	g.closed = true
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/bytes.go

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bytes
+
+import (
+	"encoding/binary"
+	"unsafe"
+)
+
+// Raw returns just the bytes without any assumptions about layout
+type Raw interface {
+	Raw() *[]byte
+}
+
+// Reader used to read various data sizes in the byte array
+type Reader interface {
+	Read8(pos int) uint8
+	Read16(pos int) uint16
+	Read32(pos int) uint32
+	Read64(pos int) uint64
+	Len() int
+}
+
+// Writer used to write various sizes of data in the byte array
+type Writer interface {
+	Write8(pos int, value uint8)
+	Write16(pos int, value uint16)
+	Write32(pos int, value uint32)
+	Write64(pos int, value uint64)
+	Len() int
+}
+
+// Bytes object for manipulating arbitrary byte arrays
+type Bytes interface {
+	Raw
+	Reader
+	Writer
+	Slice(offset int, size int) Bytes
+	LittleEndian() Bytes
+	BigEndian() Bytes
+}
+
+var nativeByteOrder binary.ByteOrder
+
+func init() {
+	buf := [2]byte{}
+	*(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0x00FF)
+
+	switch buf {
+	case [2]byte{0xFF, 0x00}:
+		nativeByteOrder = binary.LittleEndian
+	case [2]byte{0x00, 0xFF}:
+		nativeByteOrder = binary.BigEndian
+	default:
+		panic("Unable to infer byte order")
+	}
+}
+
+// New raw bytearray
+func New(data *[]byte) Bytes {
+	return (*native)(data)
+}
+
+// NewLittleEndian little endian ordering of bytes
+func NewLittleEndian(data *[]byte) Bytes {
+	if nativeByteOrder == binary.LittleEndian {
+		return (*native)(data)
+	}
+
+	return (*swapbo)(data)
+}
+
+// NewBigEndian big endian ordering of bytes
+func NewBigEndian(data *[]byte) Bytes {
+	if nativeByteOrder == binary.BigEndian {
+		return (*native)(data)
+	}
+
+	return (*swapbo)(data)
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/native.go

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bytes
+
+import (
+	"unsafe"
+)
+
+type native []byte
+
+var _ Bytes = (*native)(nil)
+
+func (b *native) Read8(pos int) uint8 {
+	return (*b)[pos]
+}
+
+func (b *native) Read16(pos int) uint16 {
+	return *(*uint16)(unsafe.Pointer(&((*b)[pos])))
+}
+
+func (b *native) Read32(pos int) uint32 {
+	return *(*uint32)(unsafe.Pointer(&((*b)[pos])))
+}
+
+func (b *native) Read64(pos int) uint64 {
+	return *(*uint64)(unsafe.Pointer(&((*b)[pos])))
+}
+
+func (b *native) Write8(pos int, value uint8) {
+	(*b)[pos] = value
+}
+
+func (b *native) Write16(pos int, value uint16) {
+	*(*uint16)(unsafe.Pointer(&((*b)[pos]))) = value
+}
+
+func (b *native) Write32(pos int, value uint32) {
+	*(*uint32)(unsafe.Pointer(&((*b)[pos]))) = value
+}
+
+func (b *native) Write64(pos int, value uint64) {
+	*(*uint64)(unsafe.Pointer(&((*b)[pos]))) = value
+}
+
+func (b *native) Slice(offset int, size int) Bytes {
+	nb := (*b)[offset : offset+size]
+	return &nb
+}
+
+func (b *native) LittleEndian() Bytes {
+	return NewLittleEndian((*[]byte)(b))
+}
+
+func (b *native) BigEndian() Bytes {
+	return NewBigEndian((*[]byte)(b))
+}
+
+func (b *native) Raw() *[]byte {
+	return (*[]byte)(b)
+}
+
+func (b *native) Len() int {
+	return len(*b)
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/swapbo.go

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package bytes
+
+import (
+	"unsafe"
+)
+
+type swapbo []byte
+
+var _ Bytes = (*swapbo)(nil)
+
+func (b *swapbo) Read8(pos int) uint8 {
+	return (*b)[pos]
+}
+
+func (b *swapbo) Read16(pos int) uint16 {
+	buf := [2]byte{}
+	buf[0] = (*b)[pos+1]
+	buf[1] = (*b)[pos+0]
+	return *(*uint16)(unsafe.Pointer(&buf[0]))
+}
+
+func (b *swapbo) Read32(pos int) uint32 {
+	buf := [4]byte{}
+	buf[0] = (*b)[pos+3]
+	buf[1] = (*b)[pos+2]
+	buf[2] = (*b)[pos+1]
+	buf[3] = (*b)[pos+0]
+	return *(*uint32)(unsafe.Pointer(&buf[0]))
+}
+
+func (b *swapbo) Read64(pos int) uint64 {
+	buf := [8]byte{}
+	buf[0] = (*b)[pos+7]
+	buf[1] = (*b)[pos+6]
+	buf[2] = (*b)[pos+5]
+	buf[3] = (*b)[pos+4]
+	buf[4] = (*b)[pos+3]
+	buf[5] = (*b)[pos+2]
+	buf[6] = (*b)[pos+1]
+	buf[7] = (*b)[pos+0]
+	return *(*uint64)(unsafe.Pointer(&buf[0]))
+}
+
+func (b *swapbo) Write8(pos int, value uint8) {
+	(*b)[pos] = value
+}
+
+func (b *swapbo) Write16(pos int, value uint16) {
+	buf := [2]byte{}
+	*(*uint16)(unsafe.Pointer(&buf[0])) = value
+	(*b)[pos+0] = buf[1]
+	(*b)[pos+1] = buf[0]
+}
+
+func (b *swapbo) Write32(pos int, value uint32) {
+	buf := [4]byte{}
+	*(*uint32)(unsafe.Pointer(&buf[0])) = value
+	(*b)[pos+0] = buf[3]
+	(*b)[pos+1] = buf[2]
+	(*b)[pos+2] = buf[1]
+	(*b)[pos+3] = buf[0]
+}
+
+func (b *swapbo) Write64(pos int, value uint64) {
+	buf := [8]byte{}
+	*(*uint64)(unsafe.Pointer(&buf[0])) = value
+	(*b)[pos+0] = buf[7]
+	(*b)[pos+1] = buf[6]
+	(*b)[pos+2] = buf[5]
+	(*b)[pos+3] = buf[4]
+	(*b)[pos+4] = buf[3]
+	(*b)[pos+5] = buf[2]
+	(*b)[pos+6] = buf[1]
+	(*b)[pos+7] = buf[0]
+}
+
+func (b *swapbo) Slice(offset int, size int) Bytes {
+	nb := (*b)[offset : offset+size]
+	return &nb
+}
+
+func (b *swapbo) LittleEndian() Bytes {
+	return NewLittleEndian((*[]byte)(b))
+}
+
+func (b *swapbo) BigEndian() Bytes {
+	return NewBigEndian((*[]byte)(b))
+}
+
+func (b *swapbo) Raw() *[]byte {
+	return (*[]byte)(b)
+}
+
+func (b *swapbo) Len() int {
+	return len(*b)
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/config.go

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nvpci
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes"
+)
+
+const (
+	// PCICfgSpaceStandardSize represents the size in bytes of the standard config space
+	PCICfgSpaceStandardSize = 256
+	// PCICfgSpaceExtendedSize represents the size in bytes of the extended config space
+	PCICfgSpaceExtendedSize = 4096
+	// PCICapabilityListPointer represents offset for the capability list pointer
+	PCICapabilityListPointer = 0x34
+	// PCIStatusCapabilityList represents the status register bit which indicates capability list support
+	PCIStatusCapabilityList = 0x10
+	// PCIStatusBytePosition represents the position of the status register
+	PCIStatusBytePosition = 0x06
+)
+
+// ConfigSpace PCI configuration space (standard extended) file path
+type ConfigSpace struct {
+	Path string
+}
+
+// ConfigSpaceIO Interface for reading and writing raw and preconfigured values
+type ConfigSpaceIO interface {
+	bytes.Bytes
+	GetVendorID() uint16
+	GetDeviceID() uint16
+	GetPCICapabilities() (*PCICapabilities, error)
+}
+
+type configSpaceIO struct {
+	bytes.Bytes
+}
+
+// PCIStandardCapability standard PCI config space
+type PCIStandardCapability struct {
+	bytes.Bytes
+}
+
+// PCIExtendedCapability extended PCI config space
+type PCIExtendedCapability struct {
+	bytes.Bytes
+	Version uint8
+}
+
+// PCICapabilities combines the standard and extended config space
+type PCICapabilities struct {
+	Standard map[uint8]*PCIStandardCapability
+	Extended map[uint16]*PCIExtendedCapability
+}
+
+func (cs *ConfigSpace) Read() (ConfigSpaceIO, error) {
+	config, err := ioutil.ReadFile(cs.Path)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+	return &configSpaceIO{bytes.New(&config)}, nil
+}
+
+func (cs *configSpaceIO) GetVendorID() uint16 {
+	return cs.Read16(0)
+}
+
+func (cs *configSpaceIO) GetDeviceID() uint16 {
+	return cs.Read16(2)
+}
+
+func (cs *configSpaceIO) GetPCICapabilities() (*PCICapabilities, error) {
+	caps := &PCICapabilities{
+		make(map[uint8]*PCIStandardCapability),
+		make(map[uint16]*PCIExtendedCapability),
+	}
+
+	support := cs.Read8(PCIStatusBytePosition) & PCIStatusCapabilityList
+	if support == 0 {
+		return nil, fmt.Errorf("pci device does not support capability list")
+	}
+
+	soffset := cs.Read8(PCICapabilityListPointer)
+	if int(soffset) >= cs.Len() {
+		return nil, fmt.Errorf("capability list pointer out of bounds")
+	}
+
+	for soffset != 0 {
+		if soffset == 0xff {
+			return nil, fmt.Errorf("config space broken")
+		}
+		if int(soffset) >= PCICfgSpaceStandardSize {
+			return nil, fmt.Errorf("standard capability list pointer out of bounds")
+		}
+		data := cs.Read32(int(soffset))
+		id := uint8(data & 0xff)
+		caps.Standard[id] = &PCIStandardCapability{
+			cs.Slice(int(soffset), cs.Len()-int(soffset)),
+		}
+		soffset = uint8((data >> 8) & 0xff)
+	}
+
+	if cs.Len() <= PCICfgSpaceStandardSize {
+		return caps, nil
+	}
+
+	eoffset := uint16(PCICfgSpaceStandardSize)
+	for eoffset != 0 {
+		if eoffset == 0xffff {
+			return nil, fmt.Errorf("config space broken")
+		}
+		if int(eoffset) >= PCICfgSpaceExtendedSize {
+			return nil, fmt.Errorf("extended capability list pointer out of bounds")
+		}
+		data := cs.Read32(int(eoffset))
+		id := uint16(data & 0xffff)
+		version := uint8((data >> 16) & 0xf)
+		caps.Extended[id] = &PCIExtendedCapability{
+			cs.Slice(int(eoffset), cs.Len()-int(eoffset)),
+			version,
+		}
+		eoffset = uint16((data >> 4) & 0xffc)
+	}
+
+	return caps, nil
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mmio.go

@@ -0,0 +1,127 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package mmio
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+	"unsafe"
+
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes"
+)
+
+// Mmio memory map a region
+type Mmio interface {
+	bytes.Raw
+	bytes.Reader
+	bytes.Writer
+	Sync() error
+	Close() error
+	Slice(offset int, size int) Mmio
+	LittleEndian() Mmio
+	BigEndian() Mmio
+}
+
+type mmio struct {
+	bytes.Bytes
+}
+
+func open(path string, offset int, size int, flags int) (Mmio, error) {
+	var mmapFlags int
+	switch flags {
+	case os.O_RDONLY:
+		mmapFlags = syscall.PROT_READ
+	case os.O_RDWR:
+		mmapFlags = syscall.PROT_READ | syscall.PROT_WRITE
+	default:
+		return nil, fmt.Errorf("invalid flags: %v", flags)
+	}
+
+	file, err := os.OpenFile(path, flags, 0)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file: %v", err)
+	}
+	defer file.Close()
+
+	fi, err := file.Stat()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get file info: %v", err)
+	}
+
+	if size > int(fi.Size()) {
+		return nil, fmt.Errorf("requested size larger than file size")
+	}
+
+	if size < 0 {
+		size = int(fi.Size())
+	}
+
+	mmap, err := syscall.Mmap(
+		int(file.Fd()),
+		int64(offset),
+		size,
+		mmapFlags,
+		syscall.MAP_SHARED)
+	if err != nil {
+		return nil, fmt.Errorf("failed to mmap file: %v", err)
+	}
+
+	return &mmio{bytes.New(&mmap)}, nil
+}
+
+// OpenRO open region readonly
+func OpenRO(path string, offset int, size int) (Mmio, error) {
+	return open(path, offset, size, os.O_RDONLY)
+}
+
+// OpenRW open region read write
+func OpenRW(path string, offset int, size int) (Mmio, error) {
+	return open(path, offset, size, os.O_RDWR)
+}
+
+func (m *mmio) Slice(offset int, size int) Mmio {
+	return &mmio{m.Bytes.Slice(offset, size)}
+}
+
+func (m *mmio) LittleEndian() Mmio {
+	return &mmio{m.Bytes.LittleEndian()}
+}
+
+func (m *mmio) BigEndian() Mmio {
+	return &mmio{m.Bytes.BigEndian()}
+}
+
+func (m *mmio) Close() error {
+	err := syscall.Munmap(*m.Bytes.Raw())
+	if err != nil {
+		return fmt.Errorf("failed to munmap file: %v", err)
+	}
+	return nil
+}
+
+func (m *mmio) Sync() error {
+	_, _, errno := syscall.Syscall(
+		syscall.SYS_MSYNC,
+		uintptr(unsafe.Pointer(&(*m.Bytes.Raw())[0])),
+		uintptr(m.Len()),
+		uintptr(syscall.MS_SYNC|syscall.MS_INVALIDATE))
+	if errno != 0 {
+		return fmt.Errorf("failed to msync file: %v", errno)
+	}
+	return nil
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mock.go

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package mmio
+
+import (
+	"fmt"
+
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes"
+)
+
+type mockMmio struct {
+	mmio
+	source *[]byte
+	offset int
+	rw     bool
+}
+
+func mockOpen(source *[]byte, offset int, size int, rw bool) (Mmio, error) {
+	if size < 0 {
+		size = len(*source) - offset
+	}
+	if (offset + size) > len(*source) {
+		return nil, fmt.Errorf("offset+size out of range")
+	}
+
+	data := append([]byte{}, (*source)[offset:offset+size]...)
+
+	m := &mockMmio{}
+	m.Bytes = bytes.New(&data).LittleEndian()
+	m.source = source
+	m.offset = offset
+	m.rw = rw
+
+	return m, nil
+}
+
+// MockOpenRO open read only
+func MockOpenRO(source *[]byte, offset int, size int) (Mmio, error) {
+	return mockOpen(source, offset, size, false)
+}
+
+// MockOpenRW open read write
+func MockOpenRW(source *[]byte, offset int, size int) (Mmio, error) {
+	return mockOpen(source, offset, size, true)
+}
+
+func (m *mockMmio) Close() error {
+	m = &mockMmio{}
+	return nil
+}
+
+func (m *mockMmio) Sync() error {
+	if !m.rw {
+		return fmt.Errorf("opened read-only")
+	}
+	for i := range *m.Bytes.Raw() {
+		(*m.source)[m.offset+i] = (*m.Bytes.Raw())[i]
+	}
+	return nil
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mock.go

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nvpci
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes"
+)
+
+// MockNvpci mock pci device
+type MockNvpci struct {
+	*nvpci
+}
+
+var _ Interface = (*MockNvpci)(nil)
+
+// NewMockNvpci create new mock PCI and remove old devices
+func NewMockNvpci() (mock *MockNvpci, rerr error) {
+	rootDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if rerr != nil {
+			os.RemoveAll(rootDir)
+		}
+	}()
+
+	mock = &MockNvpci{
+		NewFrom(rootDir).(*nvpci),
+	}
+
+	return mock, nil
+}
+
+// Cleanup remove the mocked PCI devices root folder
+func (m *MockNvpci) Cleanup() {
+	os.RemoveAll(m.pciDevicesRoot)
+}
+
+// AddMockA100 Create an A100 like GPU mock device
+func (m *MockNvpci) AddMockA100(address string, numaNode int) error {
+	deviceDir := filepath.Join(m.pciDevicesRoot, address)
+	err := os.MkdirAll(deviceDir, 0755)
+	if err != nil {
+		return err
+	}
+
+	vendor, err := os.Create(filepath.Join(deviceDir, "vendor"))
+	if err != nil {
+		return err
+	}
+	_, err = vendor.WriteString(fmt.Sprintf("0x%x", PCINvidiaVendorID))
+	if err != nil {
+		return err
+	}
+
+	class, err := os.Create(filepath.Join(deviceDir, "class"))
+	if err != nil {
+		return err
+	}
+	_, err = class.WriteString(fmt.Sprintf("0x%x", PCI3dControllerClass))
+	if err != nil {
+		return err
+	}
+
+	device, err := os.Create(filepath.Join(deviceDir, "device"))
+	if err != nil {
+		return err
+	}
+	_, err = device.WriteString("0x20bf")
+	if err != nil {
+		return err
+	}
+
+	numa, err := os.Create(filepath.Join(deviceDir, "numa_node"))
+	if err != nil {
+		return err
+	}
+	_, err = numa.WriteString(fmt.Sprintf("%v", numaNode))
+	if err != nil {
+		return err
+	}
+
+	config, err := os.Create(filepath.Join(deviceDir, "config"))
+	if err != nil {
+		return err
+	}
+	_data := make([]byte, PCICfgSpaceStandardSize)
+	data := bytes.New(&_data)
+	data.Write16(0, PCINvidiaVendorID)
+	data.Write16(2, uint16(0x20bf))
+	data.Write8(PCIStatusBytePosition, PCIStatusCapabilityList)
+	_, err = config.Write(*data.Raw())
+	if err != nil {
+		return err
+	}
+
+	bar0 := []uint64{0x00000000c2000000, 0x00000000c2ffffff, 0x0000000000040200}
+	resource, err := os.Create(filepath.Join(deviceDir, "resource"))
+	if err != nil {
+		return err
+	}
+	_, err = resource.WriteString(fmt.Sprintf("0x%x 0x%x 0x%x", bar0[0], bar0[1], bar0[2]))
+	if err != nil {
+		return err
+	}
+
+	pmcID := uint32(0x170000a1)
+	resource0, err := os.Create(filepath.Join(deviceDir, "resource0"))
+	if err != nil {
+		return err
+	}
+	_data = make([]byte, bar0[1]-bar0[0]+1)
+	data = bytes.New(&_data).LittleEndian()
+	data.Write32(0, pmcID)
+	_, err = resource0.Write(*data.Raw())
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/nvpci.go

@@ -0,0 +1,316 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nvpci
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+const (
+	// PCIDevicesRoot represents base path for all pci devices under sysfs
+	PCIDevicesRoot = "/sys/bus/pci/devices"
+	// PCINvidiaVendorID represents PCI vendor id for NVIDIA
+	PCINvidiaVendorID uint16 = 0x10de
+	// PCIVgaControllerClass represents the PCI class for VGA Controllers
+	PCIVgaControllerClass uint32 = 0x030000
+	// PCI3dControllerClass represents the PCI class for 3D Graphics accellerators
+	PCI3dControllerClass uint32 = 0x030200
+	// PCINvSwitchClass represents the PCI class for NVSwitches
+	PCINvSwitchClass uint32 = 0x068000
+)
+
+// Interface allows us to get a list of all NVIDIA PCI devices
+type Interface interface {
+	GetAllDevices() ([]*NvidiaPCIDevice, error)
+	Get3DControllers() ([]*NvidiaPCIDevice, error)
+	GetVGAControllers() ([]*NvidiaPCIDevice, error)
+	GetNVSwitches() ([]*NvidiaPCIDevice, error)
+	GetGPUs() ([]*NvidiaPCIDevice, error)
+}
+
+// MemoryResources a more human readable handle
+type MemoryResources map[int]*MemoryResource
+
+// ResourceInterface exposes some higher level functions of resources
+type ResourceInterface interface {
+	GetTotalAddressableMemory(bool) (uint64, uint64)
+}
+
+type nvpci struct {
+	pciDevicesRoot string
+}
+
+var _ Interface = (*nvpci)(nil)
+var _ ResourceInterface = (*MemoryResources)(nil)
+
+// NvidiaPCIDevice represents a PCI device for an NVIDIA product
+type NvidiaPCIDevice struct {
+	Path      string
+	Address   string
+	Vendor    uint16
+	Class     uint32
+	Device    uint16
+	NumaNode  int
+	Config    *ConfigSpace
+	Resources MemoryResources
+}
+
+// IsVGAController if class == 0x300
+func (d *NvidiaPCIDevice) IsVGAController() bool {
+	return d.Class == PCIVgaControllerClass
+}
+
+// Is3DController if class == 0x302
+func (d *NvidiaPCIDevice) Is3DController() bool {
+	return d.Class == PCI3dControllerClass
+}
+
+// IsNVSwitch if classe == 0x068
+func (d *NvidiaPCIDevice) IsNVSwitch() bool {
+	return d.Class == PCINvSwitchClass
+}
+
+// IsGPU either VGA for older cards or 3D for newer
+func (d *NvidiaPCIDevice) IsGPU() bool {
+	return d.IsVGAController() || d.Is3DController()
+}
+
+// IsResetAvailable some devices can be reset without rebooting,
+// check if applicable
+func (d *NvidiaPCIDevice) IsResetAvailable() bool {
+	_, err := os.Stat(path.Join(d.Path, "reset"))
+	return err == nil
+}
+
+// Reset perform a reset to apply a new configuration at HW level
+func (d *NvidiaPCIDevice) Reset() error {
+	err := ioutil.WriteFile(path.Join(d.Path, "reset"), []byte("1"), 0)
+	if err != nil {
+		return fmt.Errorf("unable to write to reset file: %v", err)
+	}
+	return nil
+}
+
+// New interface that allows us to get a list of all NVIDIA PCI devices
+func New() Interface {
+	return &nvpci{PCIDevicesRoot}
+}
+
+// NewFrom interface allows us to get a list of all NVIDIA PCI devices at a specific root directory
+func NewFrom(root string) Interface {
+	return &nvpci{root}
+}
+
+// GetAllDevices returns all Nvidia PCI devices on the system
+func (p *nvpci) GetAllDevices() ([]*NvidiaPCIDevice, error) {
+	deviceDirs, err := ioutil.ReadDir(p.pciDevicesRoot)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read PCI bus devices: %v", err)
+	}
+
+	var nvdevices []*NvidiaPCIDevice
+	for _, deviceDir := range deviceDirs {
+		devicePath := path.Join(p.pciDevicesRoot, deviceDir.Name())
+		nvdevice, err := NewDevice(devicePath)
+		if err != nil {
+			return nil, fmt.Errorf("error constructing NVIDIA PCI device %s: %v", deviceDir.Name(), err)
+		}
+		if nvdevice == nil {
+			continue
+		}
+		nvdevices = append(nvdevices, nvdevice)
+	}
+
+	addressToID := func(address string) uint64 {
+		address = strings.ReplaceAll(address, ":", "")
+		address = strings.ReplaceAll(address, ".", "")
+		id, _ := strconv.ParseUint(address, 16, 64)
+		return id
+	}
+
+	sort.Slice(nvdevices, func(i, j int) bool {
+		return addressToID(nvdevices[i].Address) < addressToID(nvdevices[j].Address)
+	})
+
+	return nvdevices, nil
+}
+
+// NewDevice constructs an NvidiaPCIDevice
+func NewDevice(devicePath string) (*NvidiaPCIDevice, error) {
+	address := path.Base(devicePath)
+
+	vendor, err := ioutil.ReadFile(path.Join(devicePath, "vendor"))
+	if err != nil {
+		return nil, fmt.Errorf("unable to read PCI device vendor id for %s: %v", address, err)
+	}
+	vendorStr := strings.TrimSpace(string(vendor))
+	vendorID, err := strconv.ParseUint(vendorStr, 0, 16)
+	if err != nil {
+		return nil, fmt.Errorf("unable to convert vendor string to uint16: %v", vendorStr)
+	}
+
+	if uint16(vendorID) != PCINvidiaVendorID {
+		return nil, nil
+	}
+
+	class, err := ioutil.ReadFile(path.Join(devicePath, "class"))
+	if err != nil {
+		return nil, fmt.Errorf("unable to read PCI device class for %s: %v", address, err)
+	}
+	classStr := strings.TrimSpace(string(class))
+	classID, err := strconv.ParseUint(classStr, 0, 32)
+	if err != nil {
+		return nil, fmt.Errorf("unable to convert class string to uint32: %v", classStr)
+	}
+
+	device, err := ioutil.ReadFile(path.Join(devicePath, "device"))
+	if err != nil {
+		return nil, fmt.Errorf("unable to read PCI device id for %s: %v", address, err)
+	}
+	deviceStr := strings.TrimSpace(string(device))
+	deviceID, err := strconv.ParseUint(deviceStr, 0, 16)
+	if err != nil {
+		return nil, fmt.Errorf("unable to convert device string to uint16: %v", deviceStr)
+	}
+
+	numa, err := ioutil.ReadFile(path.Join(devicePath, "numa_node"))
+	if err != nil {
+		return nil, fmt.Errorf("unable to read PCI NUMA node for %s: %v", address, err)
+	}
+	numaStr := strings.TrimSpace(string(numa))
+	numaNode, err := strconv.ParseInt(numaStr, 0, 64)
+	if err != nil {
+		return nil, fmt.Errorf("unable to convert NUMA node string to int64: %v", numaNode)
+	}
+
+	config := &ConfigSpace{
+		Path: path.Join(devicePath, "config"),
+	}
+
+	resource, err := ioutil.ReadFile(path.Join(devicePath, "resource"))
+	if err != nil {
+		return nil, fmt.Errorf("unable to read PCI resource file for %s: %v", address, err)
+	}
+
+	resources := make(map[int]*MemoryResource)
+	for i, line := range strings.Split(strings.TrimSpace(string(resource)), "\n") {
+		values := strings.Split(line, " ")
+		if len(values) != 3 {
+			return nil, fmt.Errorf("more than 3 entries in line '%d' of resource file", i)
+		}
+
+		start, _ := strconv.ParseUint(values[0], 0, 64)
+		end, _ := strconv.ParseUint(values[1], 0, 64)
+		flags, _ := strconv.ParseUint(values[2], 0, 64)
+
+		if (end - start) != 0 {
+			resources[i] = &MemoryResource{
+				uintptr(start),
+				uintptr(end),
+				flags,
+				fmt.Sprintf("%s/resource%d", devicePath, i),
+			}
+		}
+	}
+
+	nvdevice := &NvidiaPCIDevice{
+		Path:      devicePath,
+		Address:   address,
+		Vendor:    uint16(vendorID),
+		Class:     uint32(classID),
+		Device:    uint16(deviceID),
+		NumaNode:  int(numaNode),
+		Config:    config,
+		Resources: resources,
+	}
+
+	return nvdevice, nil
+}
+
+// Get3DControllers returns all NVIDIA 3D Controller PCI devices on the system
+func (p *nvpci) Get3DControllers() ([]*NvidiaPCIDevice, error) {
+	devices, err := p.GetAllDevices()
+	if err != nil {
+		return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err)
+	}
+
+	var filtered []*NvidiaPCIDevice
+	for _, d := range devices {
+		if d.Is3DController() {
+			filtered = append(filtered, d)
+		}
+	}
+
+	return filtered, nil
+}
+
+// GetVGAControllers returns all NVIDIA VGA Controller PCI devices on the system
+func (p *nvpci) GetVGAControllers() ([]*NvidiaPCIDevice, error) {
+	devices, err := p.GetAllDevices()
+	if err != nil {
+		return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err)
+	}
+
+	var filtered []*NvidiaPCIDevice
+	for _, d := range devices {
+		if d.IsVGAController() {
+			filtered = append(filtered, d)
+		}
+	}
+
+	return filtered, nil
+}
+
+// GetNVSwitches returns all NVIDIA NVSwitch PCI devices on the system
+func (p *nvpci) GetNVSwitches() ([]*NvidiaPCIDevice, error) {
+	devices, err := p.GetAllDevices()
+	if err != nil {
+		return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err)
+	}
+
+	var filtered []*NvidiaPCIDevice
+	for _, d := range devices {
+		if d.IsNVSwitch() {
+			filtered = append(filtered, d)
+		}
+	}
+
+	return filtered, nil
+}
+
+// GetGPUs returns all NVIDIA GPU devices on the system
+func (p *nvpci) GetGPUs() ([]*NvidiaPCIDevice, error) {
+	devices, err := p.GetAllDevices()
+	if err != nil {
+		return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err)
+	}
+
+	var filtered []*NvidiaPCIDevice
+	for _, d := range devices {
+		if d.IsGPU() {
+			filtered = append(filtered, d)
+		}
+	}
+
+	return filtered, nil
+}

src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/resources.go

@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nvpci
+
+import (
+	"fmt"
+	"sort"
+
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio"
+)
+
+const (
+	pmcEndianRegister = 0x4
+	pmcLittleEndian   = 0x0
+	pmcBigEndian      = 0x01000001
+)
+
+// MemoryResource represents a mmio region
+type MemoryResource struct {
+	Start uintptr
+	End   uintptr
+	Flags uint64
+	Path  string
+}
+
+// OpenRW read write mmio region
+func (mr *MemoryResource) OpenRW() (mmio.Mmio, error) {
+	rw, err := mmio.OpenRW(mr.Path, 0, int(mr.End-mr.Start+1))
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file for mmio: %v", err)
+	}
+	switch rw.Read32(pmcEndianRegister) {
+	case pmcBigEndian:
+		return rw.BigEndian(), nil
+	case pmcLittleEndian:
+		return rw.LittleEndian(), nil
+	}
+	return nil, fmt.Errorf("unknown endianness for mmio: %v", err)
+}
+
+// OpenRO read only mmio region
+func (mr *MemoryResource) OpenRO() (mmio.Mmio, error) {
+	ro, err := mmio.OpenRO(mr.Path, 0, int(mr.End-mr.Start+1))
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file for mmio: %v", err)
+	}
+	switch ro.Read32(pmcEndianRegister) {
+	case pmcBigEndian:
+		return ro.BigEndian(), nil
+	case pmcLittleEndian:
+		return ro.LittleEndian(), nil
+	}
+	return nil, fmt.Errorf("unknown endianness for mmio: %v", err)
+}
+
+// From Bit Twiddling Hacks, great resource for all low level bit manipulations
+func calcNextPowerOf2(n uint64) uint64 {
+	n--
+	n |= n >> 1
+	n |= n >> 2
+	n |= n >> 4
+	n |= n >> 8
+	n |= n >> 16
+	n |= n >> 32
+	n++
+
+	return n
+}
+
+// GetTotalAddressableMemory will accumulate the 32bit and 64bit memory windows
+// of each BAR and round the value if needed to the next power of 2; first
+// return value is the accumulated 32bit addresable memory size the second one
+// is the accumulated 64bit addressable memory size in bytes. These values are
+// needed to configure virtualized environments.
+func (mrs MemoryResources) GetTotalAddressableMemory(roundUp bool) (uint64, uint64) {
+	const pciIOVNumBAR = 6
+	const pciBaseAddressMemTypeMask = 0x06
+	const pciBaseAddressMemType32 = 0x00 /* 32 bit address */
+	const pciBaseAddressMemType64 = 0x04 /* 64 bit address */
+
+	// We need to sort the resources so the first 6 entries are the BARs
+	// How a map is represented in memory is not guaranteed, it is not an
+	// array. Keys do not have an order.
+	keys := make([]int, 0, len(mrs))
+	for k := range mrs {
+		keys = append(keys, k)
+	}
+	sort.Ints(keys)
+
+	numBAR := 0
+	memSize32bit := uint64(0)
+	memSize64bit := uint64(0)
+
+	for _, key := range keys {
+		// The PCIe spec only defines 5 BARs per device, we're
+		// discarding everything after the 5th entry of the resources
+		// file, see lspci.c
+		if key >= pciIOVNumBAR || numBAR == pciIOVNumBAR {
+			break
+		}
+		numBAR = numBAR + 1
+
+		region := mrs[key]
+
+		flags := region.Flags & pciBaseAddressMemTypeMask
+		memType32bit := flags == pciBaseAddressMemType32
+		memType64bit := flags == pciBaseAddressMemType64
+
+		memSize := (region.End - region.Start) + 1
+
+		if memType32bit {
+			memSize32bit = memSize32bit + uint64(memSize)
+		}
+		if memType64bit {
+			memSize64bit = memSize64bit + uint64(memSize)
+		}
+
+	}
+
+	if roundUp {
+		memSize32bit = calcNextPowerOf2(memSize32bit)
+		memSize64bit = calcNextPowerOf2(memSize64bit)
+	}
+
+	return memSize32bit, memSize64bit
+}

src/runtime/vendor/modules.txt

@@ -81,6 +81,7 @@ github.com/containerd/containerd/mount
 github.com/containerd/containerd/namespaces
 github.com/containerd/containerd/pkg/cri/annotations
 github.com/containerd/containerd/pkg/dialer
+github.com/containerd/containerd/pkg/ioutil
 github.com/containerd/containerd/pkg/runtimeoptions/v1
 github.com/containerd/containerd/pkg/shutdown
 github.com/containerd/containerd/pkg/ttrpcutil
@@ -305,6 +306,11 @@ github.com/vishvananda/netlink/nl
 # github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f
 ## explicit
 github.com/vishvananda/netns
+# gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965
+## explicit
+gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci
+gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes
+gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio
 # go.opencensus.io v0.23.0
 go.opencensus.io
 go.opencensus.io/internal

src/runtime/virtcontainers/acrn.go

@@ -22,10 +22,10 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"

src/runtime/virtcontainers/acrn_arch_base.go

@@ -16,7 +16,7 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/sirupsen/logrus"
 )

src/runtime/virtcontainers/acrn_arch_base_test.go

@@ -12,7 +12,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"

src/runtime/virtcontainers/acrn_test.go

@@ -14,7 +14,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"

src/runtime/virtcontainers/api.go

@@ -9,10 +9,10 @@ import (
 	"context"
 	"runtime"
 
+	deviceApi "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	deviceConfig "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
-	deviceApi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	deviceConfig "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/sirupsen/logrus"

src/runtime/virtcontainers/clh.go

@@ -32,9 +32,9 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
@@ -753,6 +753,11 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro
 	clhDisk.Readonly = &drive.ReadOnly
 	clhDisk.VhostUser = func(b bool) *bool { return &b }(false)
 
+	queues := int32(clh.config.NumVCPUs)
+	queueSize := int32(1024)
+	clhDisk.NumQueues = &queues
+	clhDisk.QueueSize = &queueSize
+
 	diskRateLimiterConfig := clh.getDiskRateLimiterConfig()
 	if diskRateLimiterConfig != nil {
 		clhDisk.SetRateLimiterConfig(*diskRateLimiterConfig)

src/runtime/virtcontainers/clh_test.go

@@ -18,7 +18,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	chclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cloud-hypervisor/client"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"

src/runtime/virtcontainers/container.go

@@ -16,10 +16,10 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
 	vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"

src/runtime/virtcontainers/container_test.go

@@ -14,11 +14,11 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"

src/runtime/virtcontainers/fc.go

@@ -24,9 +24,9 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client"
 	models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models"

src/runtime/virtcontainers/fs_share_linux.go

@@ -19,8 +19,8 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )

src/runtime/virtcontainers/hypervisor.go

@@ -16,9 +16,9 @@ import (
 
 	"github.com/pkg/errors"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 
 	"github.com/sirupsen/logrus"
@@ -370,6 +370,9 @@ type HypervisorConfig struct {
 	// VhostUserStorePathList is the list of valid values for vhost-user paths
 	VhostUserStorePathList []string
 
+	// SeccompSandbox is the qemu function which enables the seccomp feature
+	SeccompSandbox string
+
 	// KernelParams are additional guest kernel parameters.
 	KernelParams []Param
 

src/runtime/virtcontainers/interfaces.go

@@ -10,8 +10,8 @@ import (
 	"io"
 	"syscall"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"

src/runtime/virtcontainers/kata_agent.go

@@ -19,12 +19,12 @@ import (
 	"time"
 
 	"github.com/docker/go-units"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	kataclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client"

src/runtime/virtcontainers/kata_agent_test.go

@@ -23,11 +23,11 @@ import (
 	"github.com/stretchr/testify/assert"
 
 	"code.cloudfoundry.org/bytefmt"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc"

src/runtime/virtcontainers/persist.go

@@ -8,8 +8,9 @@ package virtcontainers
 import (
 	"errors"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	devconfig "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
@@ -65,7 +66,7 @@ func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState) {
 	ss.HypervisorState.BlockIndexMap = s.state.BlockIndexMap
 }
 
-func deviceToDeviceState(devices []api.Device) (dss []persistapi.DeviceState) {
+func deviceToDeviceState(devices []api.Device) (dss []devconfig.DeviceState) {
 	for _, dev := range devices {
 		dss = append(dss, dev.Save())
 	}
@@ -247,6 +248,7 @@ func (s *Sandbox) dumpConfig(ss *persistapi.SandboxState) {
 		BootFromTemplate:        sconfig.HypervisorConfig.BootFromTemplate,
 		DisableVhostNet:         sconfig.HypervisorConfig.DisableVhostNet,
 		EnableVhostUserStore:    sconfig.HypervisorConfig.EnableVhostUserStore,
+		SeccompSandbox:          sconfig.HypervisorConfig.SeccompSandbox,
 		VhostUserStorePath:      sconfig.HypervisorConfig.VhostUserStorePath,
 		VhostUserStorePathList:  sconfig.HypervisorConfig.VhostUserStorePathList,
 		GuestHookPath:           sconfig.HypervisorConfig.GuestHookPath,
@@ -323,7 +325,7 @@ func (s *Sandbox) loadAgent(as persistapi.AgentState) {
 	}
 }
 
-func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) {
+func (s *Sandbox) loadDevices(devStates []devconfig.DeviceState) {
 	s.devManager.LoadDevices(devStates)
 }
 

src/runtime/virtcontainers/persist/api/config.go

@@ -80,6 +80,9 @@ type HypervisorConfig struct {
 	// related folders, sockets and device nodes should be.
 	VhostUserStorePath string
 
+	// SeccompSandbox is the qemu function which enables the seccomp feature
+	SeccompSandbox string
+
 	// GuestHookPath is the path within the VM that will be used for 'drop-in' hooks
 	GuestHookPath string
 

src/runtime/virtcontainers/persist/api/device.go

@@ -1,124 +0,0 @@
-// Copyright (c) 2016 Intel Corporation
-// Copyright (c) 2019 Huawei Corporation
-//
-// SPDX-License-Identifier: Apache-2.0
-//
-
-package persistapi
-
-import vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
-
-// ============= sandbox level resources =============
-
-// BlockDrive represents a block storage drive which may be used in case the storage
-// driver has an underlying block storage device.
-type BlockDrive struct {
-	// File is the path to the disk-image/device which will be used with this drive
-	File string
-
-	// Format of the drive
-	Format string
-
-	// ID is used to identify this drive in the hypervisor options.
-	ID string
-
-	// MmioAddr is used to identify the slot at which the drive is attached (order?).
-	MmioAddr string
-
-	// SCSI Address of the block device, in case the device is attached using SCSI driver
-	// SCSI address is in the format SCSI-Id:LUN
-	SCSIAddr string
-
-	// NvdimmID is the nvdimm id inside the VM
-	NvdimmID string
-
-	// VirtPath at which the device appears inside the VM, outside of the container mount namespace
-	VirtPath string
-
-	// DevNo
-	DevNo string
-
-	// PCIPath is the PCI path used to identify the slot at which the drive is attached.
-	PCIPath vcTypes.PciPath
-
-	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
-	Index int
-
-	// Pmem enabled persistent memory. Use File as backing file
-	// for a nvdimm device in the guest.
-	Pmem bool
-}
-
-// VFIODev represents a VFIO drive used for hotplugging
-type VFIODev struct {
-	// ID is used to identify this drive in the hypervisor options.
-	ID string
-
-	// BDF (Bus:Device.Function) of the PCI address
-	BDF string
-
-	// Sysfsdev of VFIO mediated device
-	SysfsDev string
-
-	// Type of VFIO device
-	Type uint32
-}
-
-// VhostUserDeviceAttrs represents data shared by most vhost-user devices
-type VhostUserDeviceAttrs struct {
-	DevID      string
-	SocketPath string
-	Type       string
-
-	// MacAddress is only meaningful for vhost user net device
-	MacAddress string
-
-	// PCIPath is the PCI path used to identify the slot at which the drive is attached.
-	// It is only meaningful for vhost user block devices
-	PCIPath vcTypes.PciPath
-
-	// Block index of the device if assigned
-	Index int
-}
-
-// DeviceState is sandbox level resource which represents host devices
-// plugged to hypervisor, one Device can be shared among containers in POD
-// Refs: virtcontainers/device/drivers/generic.go:GenericDevice
-type DeviceState struct {
-	// DriverOptions is specific options for each device driver
-	// for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk"
-	DriverOptions map[string]string
-
-	// VhostUserDeviceAttrs is specific for vhost-user device driver
-	VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"`
-
-	// BlockDrive is specific for block device driver
-	BlockDrive *BlockDrive `json:",omitempty"`
-
-	ID string
-
-	// Type is used to specify driver type
-	// Refs: virtcontainers/device/config/config.go:DeviceType
-	Type string
-
-	// Type of device: c, b, u or p
-	// c , u - character(unbuffered)
-	// p - FIFO
-	// b - block(buffered) special file
-	// More info in mknod(1).
-	DevType string
-
-	// VFIODev is specific VFIO device driver
-	VFIODevs []*VFIODev `json:",omitempty"`
-
-	RefCount    uint
-	AttachCount uint
-
-	// Major, minor numbers for device.
-	Major int64
-	Minor int64
-
-	// ColdPlug specifies whether the device must be cold plugged (true)
-	// or hot plugged (false).
-	ColdPlug bool
-}

src/runtime/virtcontainers/persist/api/sandbox.go

@@ -7,6 +7,7 @@
 package persistapi
 
 import (
+	dev "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 )
 
@@ -26,7 +27,7 @@ type SandboxState struct {
 	CgroupPaths map[string]string
 
 	// Devices plugged to sandbox(hypervisor)
-	Devices []DeviceState
+	Devices []dev.DeviceState
 
 	// State is sandbox running status
 	State string

src/runtime/virtcontainers/persist_test.go

@@ -10,8 +10,8 @@ import (
 	"os"
 	"testing"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"

src/runtime/virtcontainers/physical_endpoint.go

@@ -15,9 +15,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/safchain/ethtool"

src/runtime/virtcontainers/pkg/vcmock/sandbox.go

@@ -11,9 +11,9 @@ import (
 	"io"
 	"syscall"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"

src/runtime/virtcontainers/pkg/vcmock/types.go

@@ -10,9 +10,9 @@ import (
 	"io"
 	"syscall"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	specs "github.com/opencontainers/runtime-spec/specs-go"

src/runtime/virtcontainers/qemu.go

@@ -33,12 +33,12 @@ import (
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
 	hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/uuid"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
@@ -629,30 +629,32 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 		// some devices configuration may also change kernel params, make sure this is called afterwards
 		Params: q.kernelParameters(),
 	}
+	q.checkBpfEnabled()
 
 	qemuConfig := govmmQemu.Config{
-		Name:        fmt.Sprintf("sandbox-%s", q.id),
-		UUID:        q.state.UUID,
-		Path:        qemuPath,
-		Ctx:         q.qmpMonitorCh.ctx,
-		Uid:         q.config.Uid,
-		Gid:         q.config.Gid,
-		Groups:      q.config.Groups,
-		Machine:     machine,
-		SMP:         smp,
-		Memory:      memory,
-		Devices:     devices,
-		CPUModel:    cpuModel,
-		Kernel:      kernel,
-		RTC:         rtc,
-		QMPSockets:  qmpSockets,
-		Knobs:       knobs,
-		Incoming:    incoming,
-		VGA:         "none",
-		GlobalParam: "kvm-pit.lost_tick_policy=discard",
-		Bios:        firmwarePath,
-		PFlash:      pflash,
-		PidFile:     filepath.Join(q.config.VMStorePath, q.id, "pid"),
+		Name:           fmt.Sprintf("sandbox-%s", q.id),
+		UUID:           q.state.UUID,
+		Path:           qemuPath,
+		Ctx:            q.qmpMonitorCh.ctx,
+		Uid:            q.config.Uid,
+		Gid:            q.config.Gid,
+		Groups:         q.config.Groups,
+		Machine:        machine,
+		SMP:            smp,
+		Memory:         memory,
+		Devices:        devices,
+		CPUModel:       cpuModel,
+		SeccompSandbox: q.config.SeccompSandbox,
+		Kernel:         kernel,
+		RTC:            rtc,
+		QMPSockets:     qmpSockets,
+		Knobs:          knobs,
+		Incoming:       incoming,
+		VGA:            "none",
+		GlobalParam:    "kvm-pit.lost_tick_policy=discard",
+		Bios:           firmwarePath,
+		PFlash:         pflash,
+		PidFile:        filepath.Join(q.config.VMStorePath, q.id, "pid"),
 	}
 
 	qemuConfig.Devices, qemuConfig.Bios, err = q.arch.appendProtectionDevice(qemuConfig.Devices, firmwarePath, firmwareVolumePath)
@@ -679,8 +681,10 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 	// Add PCIe Root Port devices to hypervisor
 	// The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port.
 	// For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt
+	memSize32bit, memSize64bit := q.arch.getBARsMaxAddressableMemory()
+
 	if hypervisorConfig.PCIeRootPort > 0 {
-		qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort)
+		qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort, memSize32bit, memSize64bit)
 	}
 
 	q.qemuConfig = qemuConfig
@@ -689,6 +693,25 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi
 	return err
 }
 
+func (q *qemu) checkBpfEnabled() {
+	if q.config.SeccompSandbox != "" {
+		out, err := os.ReadFile("/proc/sys/net/core/bpf_jit_enable")
+		if err != nil {
+			q.Logger().WithError(err).Warningf("failed to get bpf_jit_enable status")
+			return
+		}
+		enabled, err := strconv.Atoi(string(out))
+		if err != nil {
+			q.Logger().WithError(err).Warningf("failed to convert bpf_jit_enable status to integer")
+			return
+		}
+		if enabled == 0 {
+			q.Logger().Warningf("bpf_jit_enable is disabled. " +
+				"It's recommended to turn on bpf_jit_enable to reduce the performance impact of QEMU seccomp sandbox.")
+		}
+	}
+}
+
 func (q *qemu) vhostFSSocketPath(id string) (string, error) {
 	return utils.BuildSocketPath(q.config.VMStorePath, id, vhostFSSocket)
 }
@@ -1341,7 +1364,9 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri
 			return err
 		}
 
-		if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil {
+		queues := int(q.config.NumVCPUs)
+
+		if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, queues, true, defaultDisableModern); err != nil {
 			return err
 		}
 	case q.config.BlockDeviceDriver == config.VirtioBlockCCW:
@@ -2352,7 +2377,7 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff
 }
 
 // genericAppendPCIeRootPort appends to devices the given pcie-root-port
-func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device {
+func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string, memSize32bit uint64, memSize64bit uint64) []govmmQemu.Device {
 	var (
 		bus           string
 		chassis       string
@@ -2378,6 +2403,8 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin
 				Slot:          strconv.FormatUint(uint64(i), 10),
 				Multifunction: multiFunction,
 				Addr:          addr,
+				MemReserve:    fmt.Sprintf("%dB", memSize32bit),
+				Pref64Reserve: fmt.Sprintf("%dB", memSize64bit),
 			},
 		)
 	}

src/runtime/virtcontainers/qemu_arch_base.go

@@ -18,8 +18,9 @@ import (
 	"strings"
 
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 )
@@ -138,7 +139,7 @@ type qemuArch interface {
 	setIgnoreSharedMemoryMigrationCaps(context.Context, *govmmQemu.QMP) error
 
 	// appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus
-	appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device
+	appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32, memSize32bit uint64, memSize64bit uint64) []govmmQemu.Device
 
 	// append vIOMMU device
 	appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error)
@@ -151,6 +152,10 @@ type qemuArch interface {
 	// a firmware, returns a string containing the path to the firmware that should
 	// be used with the -bios option, ommit -bios option if the path is empty.
 	appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error)
+
+	// scans the PCIe space and returns the biggest BAR sizes for 32-bit
+	// and 64-bit addressable memory
+	getBARsMaxAddressableMemory() (uint64, uint64)
 }
 
 type qemuArchBase struct {
@@ -787,8 +792,39 @@ func (q *qemuArchBase) addBridge(b types.Bridge) {
 }
 
 // appendPCIeRootPortDevice appends to devices the given pcie-root-port
-func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device {
-	return genericAppendPCIeRootPort(devices, number, q.qemuMachine.Type)
+func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32, memSize32bit uint64, memSize64bit uint64) []govmmQemu.Device {
+	return genericAppendPCIeRootPort(devices, number, q.qemuMachine.Type, memSize32bit, memSize64bit)
+}
+
+func (q *qemuArchBase) getBARsMaxAddressableMemory() (uint64, uint64) {
+
+	pci := nvpci.New()
+	devs, _ := pci.GetAllDevices()
+
+	// Since we do not know which devices are going to be hotplugged,
+	// we're going to use the GPU with the biggest BARs to initialize the
+	// root port, this should work for all other devices as well.
+	// defaults are 2MB for both, if no suitable devices found
+	max32bit := uint64(2 * 1024 * 1024)
+	max64bit := uint64(2 * 1024 * 1024)
+
+	for _, dev := range devs {
+		if !dev.IsGPU() {
+			continue
+		}
+		memSize32bit, memSize64bit := dev.Resources.GetTotalAddressableMemory(true)
+		if max32bit < memSize32bit {
+			max32bit = memSize32bit
+		}
+		if max64bit < memSize64bit {
+			max64bit = memSize64bit
+		}
+	}
+	// The actual 32bit is most of the time a power of 2 but we need some
+	// buffer so double that to leave space for other IO functions.
+	// The 64bit size is not a power of 2 and hence is already rounded up
+	// to the higher value.
+	return max32bit * 2, max64bit
 }
 
 // appendIOMMU appends a virtual IOMMU device

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -19,7 +19,7 @@ import (
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/pkg/errors"

src/runtime/virtcontainers/qemu_s390x.go

@@ -13,8 +13,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"

src/runtime/virtcontainers/qemu_s390x_test.go

@@ -16,7 +16,7 @@ import (
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 )
 
 func newTestQemu(assert *assert.Assertions, machineType string) qemuArch {

src/runtime/virtcontainers/qemu_test.go

@@ -15,9 +15,9 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/govmm"
 	govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"

src/runtime/virtcontainers/sandbox.go

@@ -25,12 +25,12 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/api"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
+	deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
-	deviceManager "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"

src/runtime/virtcontainers/sandbox_test.go

@@ -16,10 +16,10 @@ import (
 	"syscall"
 	"testing"
 
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager"
 	ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers"
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs"
 

src/runtime/virtcontainers/vhostuser_endpoint.go

@@ -14,7 +14,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
+	"github.com/kata-containers/kata-containers/src/runtime/pkg/device/config"
 	persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api"
 	vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils"

tools/osbuilder/rootfs-builder/rootfs.sh

@@ -540,8 +540,13 @@ EOF
 
 	if [ -f "$chrony_systemd_service" ]; then
 		# Remove user option, user could not exist in the rootfs
+		# Set the /var/lib/chrony for ReadWritePaths to be ignored if
+		# its nonexistent, this broke the service on boot previously
+		# due to the directory not being present "(code=exited, status=226/NAMESPACE)"
 		sed -i -e 's/^\(ExecStart=.*\)-u [[:alnum:]]*/\1/g' \
-		       -e '/^\[Unit\]/a ConditionPathExists=\/dev\/ptp0' ${chrony_systemd_service}
+		       -e '/^\[Unit\]/a ConditionPathExists=\/dev\/ptp0' \
+		       -e 's/^ReadWritePaths=\(.\+\) \/var\/lib\/chrony \(.\+\)$/ReadWritePaths=\1 -\/var\/lib\/chrony \2/m' \
+		       ${chrony_systemd_service}
 	fi
 
 	AGENT_DIR="${ROOTFS_DIR}/usr/bin"

tools/packaging/kernel/README.md

@@ -47,7 +47,7 @@ Options:
 	-g <vendor> 	: GPU vendor, intel or nvidia.
 	-h          	: Display this help.
 	-k <path>   	: Path to kernel to build.
-	-p <path>   	: Path to a directory with patches to apply to kernel.
+	-p <path>   	: Path to a directory with patches to apply to kernel, only patches in top-level directory are applied.
 	-t <hypervisor>	: Hypervisor_target.
 	-v <version>	: Kernel version to use if kernel path not provided.
 ```
@@ -76,7 +76,7 @@ $ ./build-kernel.sh setup
 The script `./build-kernel.sh` tries to apply the patches from
 `${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/patches/` when it
 sets up a kernel. If you want to add a source modification, add a patch on this
-directory.
+directory. Patches present in the top-level directory are applied, with subdirectories being ignored.
 
 The script also adds a kernel config file from
 `${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/configs/` to `.config`

tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf

@@ -1,5 +1,6 @@
 # Intel Trust Domain Extensions (Intel TDX)
 
+CONFIG_EFI=y
 CONFIG_EFI_STUB=y
 CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y
 CONFIG_INTEL_TDX_GUEST=y

tools/packaging/kernel/kata_config_version

@@ -1 +1 @@
-92
+93

tools/packaging/release/publish-kata-image.sh

@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-#Copyright (c) 2018 Intel Corporation
-#
-#SPDX-License-Identifier: Apache-2.0
-#
-
-[ -z "${DEBUG}" ] || set -x
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-workdir="${PWD}"
-
-readonly script_name="$(basename "${BASH_SOURCE[0]}")"
-readonly script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-readonly project="kata-containers"
-GOPATH=${GOPATH:-${HOME}/go}
-
-source "${script_dir}/../scripts/lib.sh"
-source "${script_dir}/../obs-packaging/scripts/pkglib.sh"
-
-die() {
-	msg="$*"
-	echo "ERROR: ${FUNCNAME[1]} ${msg}" >&2
-	exit 1
-}
-
-usage() {
-	return_code=${1:-0}
-	cat <<EOF
-Usage:
-
-${script_name} [options]  <version>
-
-version: Kata version to create the image.
-
-Create image for a kata version.
-
-options:
-
--h      : show this help
--p      : push image to github
-EOF
-
-	exit "${return_code}"
-}
-
-main() {
-	push="false"
-	while getopts "d:hp" opt; do
-		case $opt in
-		h) usage 0 ;;
-		p) push="true" ;;
-		esac
-	done
-
-	shift $((OPTIND - 1))
-	kata_version=${1:-}
-	[ -n "${kata_version}" ] || usage "1"
-
-	ref="refs/tags/${kata_version}^{}"
-	agent_sha=$(get_kata_hash "agent" "${ref}")
-	agent_sha=${agent_sha:0:${short_commit_length}}
-	image_tarball=$(find -name 'kata-containers-*.tar.gz' | grep "${kata_version}" | grep "${agent_sha}") ||
-		"${script_dir}/../obs-packaging/kata-containers-image/build_image.sh" -v "${kata_version}"
-	image_tarball=$(find -name 'kata-containers-*.tar.gz' | grep "${kata_version}" | grep "${agent_sha}" ) || die "file not found ${image_tarball}"
-
-	if [ ${push} == "true" ]; then
-		hub -C "${GOPATH}/src/github.com/${project}/agent" release edit -a "${image_tarball}" "${kata_version}"
-	else
-		echo "Wont push image to github use -p option to do it."
-	fi
-}
-
-main $@

tools/packaging/scripts/apply_patches.sh

@@ -36,7 +36,7 @@ fi
 
 echo "INFO: Apply patches from $patches_dir"
 if [ -d "$patches_dir" ]; then
-	patches=($(find "$patches_dir" -name '*.patch'|sort -t- -k1,1n))
+	patches=($(find "$patches_dir" -maxdepth 1 -name '*.patch'|sort -t- -k1,1n))
 	echo "INFO: Found ${#patches[@]} patches"
 	for patch in ${patches[@]}; do
 		echo "INFO: Apply $patch"

versions.yaml

@@ -153,7 +153,7 @@ assets:
   kernel:
     description: "Linux kernel optimised for virtual machines"
     url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/"
-    version: "v5.15.26"
+    version: "v5.15.48"
     tdx:
       description: "Linux kernel that supports TDX"
       url: "https://github.com/intel/tdx/archive/refs/tags"