diff --git a/ci/darwin-test.sh b/ci/darwin-test.sh index 92317653be..f9d32e743d 100755 --- a/ci/darwin-test.sh +++ b/ci/darwin-test.sh @@ -11,10 +11,10 @@ runtimedir=$cidir/../src/runtime build_working_packages() { # working packages: - device_api=$runtimedir/virtcontainers/device/api - device_config=$runtimedir/virtcontainers/device/config - device_drivers=$runtimedir/virtcontainers/device/drivers - device_manager=$runtimedir/virtcontainers/device/manager + device_api=$runtimedir/pkg/device/api + device_config=$runtimedir/pkg/device/config + device_drivers=$runtimedir/pkg/device/drivers + device_manager=$runtimedir/pkg/device/manager rc_pkg_dir=$runtimedir/pkg/resourcecontrol/ utils_pkg_dir=$runtimedir/virtcontainers/utils diff --git a/docs/how-to/run-kata-with-k8s.md b/docs/how-to/run-kata-with-k8s.md index fd53838b88..baee63bffe 100644 --- a/docs/how-to/run-kata-with-k8s.md +++ b/docs/how-to/run-kata-with-k8s.md @@ -99,7 +99,18 @@ $ sudo systemctl restart kubelet $ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /var/run/crio/crio.sock --pod-network-cidr=10.244.0.0/16 # If using containerd -$ sudo kubeadm init --ignore-preflight-errors=all --cri-socket /run/containerd/containerd.sock --pod-network-cidr=10.244.0.0/16 +$ cat < "$@" ##TARGET optimize: optimized build optimize: show-summary show-header - @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) --$(BUILD_TYPE) $(EXTRA_RUSTFEATURES) + @RUSTFLAGS="-C link-arg=-s $(EXTRA_RUSTFLAGS) --deny warnings" cargo build --target $(TRIPLE) $(if $(findstring release,$(BUILD_TYPE)),--release) $(EXTRA_RUSTFEATURES) ##TARGET install: install agent install: install-services diff --git a/src/runtime/go.mod b/src/runtime/go.mod index f5fb414145..2591db9d5b 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -44,6 +44,7 @@ require ( github.com/urfave/cli v1.22.2 github.com/vishvananda/netlink v1.1.1-0.20210924202909-187053b97868 github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f + gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 go.opentelemetry.io/otel v1.3.0 go.opentelemetry.io/otel/exporters/jaeger v1.0.0 go.opentelemetry.io/otel/sdk v1.3.0 diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 891838cc84..60c8ddf41f 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -689,6 +689,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 h1:EXE1ZsUqiUWGV5Dw2oTYpXx24ffxj0//yhTB0Ppv+4s= +gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965/go.mod h1:TBB3sR7/jg4RCThC/cgT4fB8mAbbMO307TycfgeR59w= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= diff --git a/src/runtime/pkg/containerd-shim-v2/service.go b/src/runtime/pkg/containerd-shim-v2/service.go index 27ebe19268..9e703c9e21 100644 --- a/src/runtime/pkg/containerd-shim-v2/service.go +++ b/src/runtime/pkg/containerd-shim-v2/service.go @@ -7,6 +7,7 @@ package containerdshim import ( "context" + "fmt" "io" "os" sysexec "os/exec" @@ -85,6 +86,11 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu vci.SetLogger(ctx, shimLog) katautils.SetLogger(ctx, shimLog, shimLog.Logger.Level) + ns, found := namespaces.Namespace(ctx) + if !found { + return nil, fmt.Errorf("shim namespace cannot be empty") + } + s := &service{ id: id, pid: uint32(os.Getpid()), @@ -93,6 +99,7 @@ func New(ctx context.Context, id string, publisher cdshim.Publisher, shutdown fu events: make(chan interface{}, chSize), ec: make(chan exit, bufferSize), cancel: shutdown, + namespace: ns, } go s.processExits() @@ -131,6 +138,9 @@ type service struct { id string + // Namespace from upper container engine + namespace string + mu sync.Mutex eventSendMu sync.Mutex diff --git a/src/runtime/pkg/containerd-shim-v2/shim_io_binary.go b/src/runtime/pkg/containerd-shim-v2/shim_io_binary.go new file mode 100644 index 0000000000..39a72ec115 --- /dev/null +++ b/src/runtime/pkg/containerd-shim-v2/shim_io_binary.go @@ -0,0 +1,216 @@ +// Copyright (c) 2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +package containerdshim + +import ( + "context" + "fmt" + "io" + "net/url" + "os" + "syscall" + "time" + + "golang.org/x/sys/execabs" + + "github.com/hashicorp/go-multierror" +) + +const ( + binaryIOProcTermTimeout = 12 * time.Second // Give logger process solid 10 seconds for cleanup +) + +var ( + _ IO = &binaryIO{} +) + +// binaryIO related code is from https://github.com/containerd/containerd/blob/v1.6.6/pkg/process/io.go#L311 +type binaryIO struct { + cmd *execabs.Cmd + out, err *pipe +} + +// https://github.com/containerd/containerd/blob/v1.6.6/pkg/process/io.go#L248 +func newBinaryIO(ctx context.Context, ns, id string, uri *url.URL) (bio *binaryIO, err error) { + var closers []func() error + defer func() { + if err == nil { + return + } + result := multierror.Append(err) + for _, fn := range closers { + result = multierror.Append(result, fn()) + } + err = multierror.Flatten(result) + }() + + out, err := newPipe() + if err != nil { + return nil, fmt.Errorf("failed to create stdout pipes: %w", err) + } + closers = append(closers, out.Close) + + serr, err := newPipe() + if err != nil { + return nil, fmt.Errorf("failed to create stderr pipes: %w", err) + } + closers = append(closers, serr.Close) + + r, w, err := os.Pipe() + if err != nil { + return nil, err + } + closers = append(closers, r.Close, w.Close) + + cmd := newBinaryCmd(uri, id, ns) + cmd.ExtraFiles = append(cmd.ExtraFiles, out.r, serr.r, w) + // don't need to register this with the reaper or wait when + // running inside a shim + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("failed to start binary process: %w", err) + } + closers = append(closers, func() error { return cmd.Process.Kill() }) + + // close our side of the pipe after start + if err := w.Close(); err != nil { + return nil, fmt.Errorf("failed to close write pipe after start: %w", err) + } + + // wait for the logging binary to be ready + b := make([]byte, 1) + if _, err := r.Read(b); err != nil && err != io.EOF { + return nil, fmt.Errorf("failed to read from logging binary: %w", err) + } + + return &binaryIO{ + cmd: cmd, + out: out, + err: serr, + }, nil +} + +// newBinaryCmd returns a Cmd to be used to start a logging binary. +// The Cmd is generated from the provided uri, and the container ID and +// namespace are appended to the Cmd environment. +func newBinaryCmd(binaryURI *url.URL, id, ns string) *execabs.Cmd { + var args []string + for k, vs := range binaryURI.Query() { + args = append(args, k) + if len(vs) > 0 { + args = append(args, vs[0]) + } + } + + cmd := execabs.Command(binaryURI.Path, args...) + + cmd.Env = append(cmd.Env, + "CONTAINER_ID="+id, + "CONTAINER_NAMESPACE="+ns, + ) + + return cmd +} + +func (bi *binaryIO) Stdin() io.ReadCloser { + return nil +} + +func (bi *binaryIO) Stdout() io.Writer { + return bi.out.w +} + +func (bi *binaryIO) Stderr() io.Writer { + return bi.err.w +} + +func (bi *binaryIO) Close() error { + var ( + result *multierror.Error + ) + + for _, v := range []*pipe{bi.out, bi.err} { + if v != nil { + if err := v.Close(); err != nil { + result = multierror.Append(result, err) + } + } + } + + if err := bi.cancel(); err != nil { + result = multierror.Append(result, err) + } + + return result.ErrorOrNil() +} + +func (bi *binaryIO) cancel() error { + if bi.cmd == nil || bi.cmd.Process == nil { + return nil + } + + // Send SIGTERM first, so logger process has a chance to flush and exit properly + if err := bi.cmd.Process.Signal(syscall.SIGTERM); err != nil { + result := multierror.Append(fmt.Errorf("failed to send SIGTERM: %w", err)) + + shimLog.WithError(err).Warn("failed to send SIGTERM signal, killing logging shim") + + if err := bi.cmd.Process.Kill(); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to kill process after faulty SIGTERM: %w", err)) + } + + return result.ErrorOrNil() + } + + done := make(chan error, 1) + go func() { + done <- bi.cmd.Wait() + }() + + select { + case err := <-done: + return err + case <-time.After(binaryIOProcTermTimeout): + shimLog.Warn("failed to wait for shim logger process to exit, killing") + + err := bi.cmd.Process.Kill() + if err != nil { + return fmt.Errorf("failed to kill shim logger process: %w", err) + } + + return nil + } +} + +func newPipe() (*pipe, error) { + r, w, err := os.Pipe() + if err != nil { + return nil, err + } + return &pipe{ + r: r, + w: w, + }, nil +} + +type pipe struct { + r *os.File + w *os.File +} + +// https://github.com/containerd/containerd/blob/v1.6.6/vendor/github.com/containerd/go-runc/io.go#L71 +func (p *pipe) Close() error { + var result *multierror.Error + + if err := p.w.Close(); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to close write pipe: %w", err)) + } + + if err := p.r.Close(); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to close read pipe: %w", err)) + } + + return multierror.Prefix(result.ErrorOrNil(), "pipe:") +} diff --git a/src/runtime/pkg/containerd-shim-v2/shim_io_file.go b/src/runtime/pkg/containerd-shim-v2/shim_io_file.go new file mode 100644 index 0000000000..8ef43981d1 --- /dev/null +++ b/src/runtime/pkg/containerd-shim-v2/shim_io_file.go @@ -0,0 +1,80 @@ +// Copyright (c) 2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +package containerdshim + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + + cioutil "github.com/containerd/containerd/pkg/ioutil" +) + +var ( + _ IO = &fileIO{} +) + +// fileIO only support write both stdout/stderr to the same file +type fileIO struct { + outw io.WriteCloser + errw io.WriteCloser + path string +} + +// openLogFile opens/creates a container log file with its directory. +func openLogFile(path string) (*os.File, error) { + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + return nil, err + } + return os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0640) +} + +func newFileIO(ctx context.Context, stdio *stdio, uri *url.URL) (*fileIO, error) { + var outw, errw, f io.WriteCloser + var err error + + logFile := uri.Path + if f, err = openLogFile(logFile); err != nil { + return nil, err + } + + if stdio.Stdout != "" { + outw = cioutil.NewSerialWriteCloser(f) + } + + if !stdio.Console && stdio.Stderr != "" { + errw = cioutil.NewSerialWriteCloser(f) + } + + return &fileIO{ + path: logFile, + outw: outw, + errw: errw, + }, nil +} + +func (fi *fileIO) Close() error { + if fi.outw != nil { + return wc(fi.outw) + } else if fi.errw != nil { + return wc(fi.errw) + } + return nil +} + +func (fi *fileIO) Stdin() io.ReadCloser { + return nil +} + +func (fi *fileIO) Stdout() io.Writer { + return fi.outw +} + +func (fi *fileIO) Stderr() io.Writer { + return fi.errw +} diff --git a/src/runtime/pkg/containerd-shim-v2/shim_io_pipe.go b/src/runtime/pkg/containerd-shim-v2/shim_io_pipe.go new file mode 100644 index 0000000000..702549b41f --- /dev/null +++ b/src/runtime/pkg/containerd-shim-v2/shim_io_pipe.go @@ -0,0 +1,95 @@ +// Copyright (c) 2022 Ant Group +// +// SPDX-License-Identifier: Apache-2.0 +// + +package containerdshim + +import ( + "context" + "fmt" + "io" + "syscall" + + "github.com/containerd/fifo" + "github.com/hashicorp/go-multierror" +) + +var ( + _ IO = &pipeIO{} +) + +type pipeIO struct { + in io.ReadCloser + outw io.WriteCloser + errw io.WriteCloser +} + +func newPipeIO(ctx context.Context, stdio *stdio) (*pipeIO, error) { + var in io.ReadCloser + var outw io.WriteCloser + var errw io.WriteCloser + var err error + + if stdio.Stdin != "" { + in, err = fifo.OpenFifo(ctx, stdio.Stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) + if err != nil { + return nil, err + } + } + + if stdio.Stdout != "" { + outw, err = fifo.OpenFifo(ctx, stdio.Stdout, syscall.O_RDWR, 0) + if err != nil { + return nil, err + } + } + + if !stdio.Console && stdio.Stderr != "" { + errw, err = fifo.OpenFifo(ctx, stdio.Stderr, syscall.O_RDWR, 0) + if err != nil { + return nil, err + } + } + + pipeIO := &pipeIO{ + in: in, + outw: outw, + errw: errw, + } + + return pipeIO, nil +} + +func (pi *pipeIO) Stdin() io.ReadCloser { + return pi.in +} + +func (pi *pipeIO) Stdout() io.Writer { + return pi.outw +} + +func (pi *pipeIO) Stderr() io.Writer { + return pi.errw +} + +func (pi *pipeIO) Close() error { + var result *multierror.Error + + if pi.in != nil { + if err := pi.in.Close(); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to close stdin: %w", err)) + } + pi.in = nil + } + + if err := wc(pi.outw); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to close stdout: %w", err)) + } + + if err := wc(pi.errw); err != nil { + result = multierror.Append(result, fmt.Errorf("failed to close stderr: %w", err)) + } + + return result.ErrorOrNil() +} diff --git a/src/runtime/pkg/containerd-shim-v2/start.go b/src/runtime/pkg/containerd-shim-v2/start.go index 65bfe6d9a1..d8c9368995 100644 --- a/src/runtime/pkg/containerd-shim-v2/start.go +++ b/src/runtime/pkg/containerd-shim-v2/start.go @@ -8,6 +8,7 @@ package containerdshim import ( "context" "fmt" + "github.com/sirupsen/logrus" "github.com/containerd/containerd/api/types/task" @@ -75,7 +76,7 @@ func startContainer(ctx context.Context, s *service, c *container) (retErr error c.stdinPipe = stdin if c.stdin != "" || c.stdout != "" || c.stderr != "" { - tty, err := newTtyIO(ctx, c.stdin, c.stdout, c.stderr, c.terminal) + tty, err := newTtyIO(ctx, s.namespace, c.id, c.stdin, c.stdout, c.stderr, c.terminal) if err != nil { return err } @@ -141,7 +142,7 @@ func startExec(ctx context.Context, s *service, containerID, execID string) (e * execs.stdinPipe = stdin - tty, err := newTtyIO(ctx, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal) + tty, err := newTtyIO(ctx, s.namespace, execs.id, execs.tty.stdin, execs.tty.stdout, execs.tty.stderr, execs.tty.terminal) if err != nil { return nil, err } diff --git a/src/runtime/pkg/containerd-shim-v2/stream.go b/src/runtime/pkg/containerd-shim-v2/stream.go index 58045359b3..c20e63de82 100644 --- a/src/runtime/pkg/containerd-shim-v2/stream.go +++ b/src/runtime/pkg/containerd-shim-v2/stream.go @@ -7,16 +7,22 @@ package containerdshim import ( "context" + "fmt" "io" + "net/url" "sync" - "syscall" - "github.com/containerd/fifo" "github.com/sirupsen/logrus" ) -// The buffer size used to specify the buffer for IO streams copy -const bufSize = 32 << 10 +const ( + // The buffer size used to specify the buffer for IO streams copy + bufSize = 32 << 10 + + shimLogPluginBinary = "binary" + shimLogPluginFifo = "fifo" + shimLogPluginFile = "file" +) var ( bufPool = sync.Pool{ @@ -27,76 +33,84 @@ var ( } ) +type stdio struct { + Stdin string + Stdout string + Stderr string + Console bool +} +type IO interface { + io.Closer + Stdin() io.ReadCloser + Stdout() io.Writer + Stderr() io.Writer +} + type ttyIO struct { - Stdin io.ReadCloser - Stdout io.Writer - Stderr io.Writer + io IO + raw *stdio } func (tty *ttyIO) close() { - - if tty.Stdin != nil { - tty.Stdin.Close() - tty.Stdin = nil - } - cf := func(w io.Writer) { - if w == nil { - return - } - if c, ok := w.(io.WriteCloser); ok { - c.Close() - } - } - cf(tty.Stdout) - cf(tty.Stderr) + tty.io.Close() } -func newTtyIO(ctx context.Context, stdin, stdout, stderr string, console bool) (*ttyIO, error) { - var in io.ReadCloser - var outw io.Writer - var errw io.Writer +// newTtyIO creates a new ttyIO struct. +// ns(namespace)/id(container ID) are used for containerd binary IO. +// containerd will pass the ns/id as ENV to the binary log driver, +// and the binary log driver will use ns/id to get the log options config file. +// for example nerdctl: https://github.com/containerd/nerdctl/blob/v0.21.0/pkg/logging/logging.go#L102 +func newTtyIO(ctx context.Context, ns, id, stdin, stdout, stderr string, console bool) (*ttyIO, error) { var err error + var io IO - if stdin != "" { - in, err = fifo.OpenFifo(ctx, stdin, syscall.O_RDONLY|syscall.O_NONBLOCK, 0) - if err != nil { - return nil, err - } + raw := &stdio{ + Stdin: stdin, + Stdout: stdout, + Stderr: stderr, + Console: console, } - if stdout != "" { - outw, err = fifo.OpenFifo(ctx, stdout, syscall.O_RDWR, 0) - if err != nil { - return nil, err - } + uri, err := url.Parse(stdout) + if err != nil { + return nil, fmt.Errorf("unable to parse stdout uri: %w", err) } - if !console && stderr != "" { - errw, err = fifo.OpenFifo(ctx, stderr, syscall.O_RDWR, 0) - if err != nil { - return nil, err - } + if uri.Scheme == "" { + uri.Scheme = "fifo" } - ttyIO := &ttyIO{ - Stdin: in, - Stdout: outw, - Stderr: errw, + switch uri.Scheme { + case shimLogPluginFifo: + io, err = newPipeIO(ctx, raw) + case shimLogPluginBinary: + io, err = newBinaryIO(ctx, ns, id, uri) + case shimLogPluginFile: + io, err = newFileIO(ctx, raw, uri) + default: + return nil, fmt.Errorf("unknown STDIO scheme %s", uri.Scheme) } - return ttyIO, nil + if err != nil { + return nil, fmt.Errorf("failed to creat io stream: %w", err) + } + + return &ttyIO{ + io: io, + raw: raw, + }, nil } func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO, stdinPipe io.WriteCloser, stdoutPipe, stderrPipe io.Reader) { var wg sync.WaitGroup - if tty.Stdin != nil { + if tty.io.Stdin() != nil { wg.Add(1) go func() { shimLog.Debug("stdin io stream copy started") p := bufPool.Get().(*[]byte) defer bufPool.Put(p) - io.CopyBuffer(stdinPipe, tty.Stdin, *p) + io.CopyBuffer(stdinPipe, tty.io.Stdin(), *p) // notify that we can close process's io safely. close(stdinCloser) wg.Done() @@ -104,30 +118,30 @@ func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO }() } - if tty.Stdout != nil { + if tty.io.Stdout() != nil { wg.Add(1) go func() { shimLog.Debug("stdout io stream copy started") p := bufPool.Get().(*[]byte) defer bufPool.Put(p) - io.CopyBuffer(tty.Stdout, stdoutPipe, *p) + io.CopyBuffer(tty.io.Stdout(), stdoutPipe, *p) wg.Done() - if tty.Stdin != nil { + if tty.io.Stdin() != nil { // close stdin to make the other routine stop - tty.Stdin.Close() + tty.io.Stdin().Close() } shimLog.Debug("stdout io stream copy exited") }() } - if tty.Stderr != nil && stderrPipe != nil { + if tty.io.Stderr() != nil && stderrPipe != nil { wg.Add(1) go func() { shimLog.Debug("stderr io stream copy started") p := bufPool.Get().(*[]byte) defer bufPool.Put(p) - io.CopyBuffer(tty.Stderr, stderrPipe, *p) + io.CopyBuffer(tty.io.Stderr(), stderrPipe, *p) wg.Done() shimLog.Debug("stderr io stream copy exited") }() @@ -138,3 +152,10 @@ func ioCopy(shimLog *logrus.Entry, exitch, stdinCloser chan struct{}, tty *ttyIO close(exitch) shimLog.Debug("all io stream copy goroutines exited") } + +func wc(w io.WriteCloser) error { + if w == nil { + return nil + } + return w.Close() +} diff --git a/src/runtime/pkg/containerd-shim-v2/stream_test.go b/src/runtime/pkg/containerd-shim-v2/stream_test.go index d5317a172a..ea4f026ca1 100644 --- a/src/runtime/pkg/containerd-shim-v2/stream_test.go +++ b/src/runtime/pkg/containerd-shim-v2/stream_test.go @@ -7,7 +7,6 @@ package containerdshim import ( "context" - "github.com/sirupsen/logrus" "io" "os" "path/filepath" @@ -15,6 +14,8 @@ import ( "testing" "time" + "github.com/sirupsen/logrus" + "github.com/containerd/fifo" "github.com/stretchr/testify/assert" ) @@ -45,7 +46,7 @@ func TestNewTtyIOFifoReopen(t *testing.T) { defer outr.Close() errr = createReadFifo(stderr) defer errr.Close() - tty, err = newTtyIO(ctx, "", stdout, stderr, false) + tty, err = newTtyIO(ctx, "", "", "", stdout, stderr, false) assert.NoError(err) defer tty.close() @@ -72,9 +73,9 @@ func TestNewTtyIOFifoReopen(t *testing.T) { } } - checkFifoWrite(tty.Stdout) + checkFifoWrite(tty.io.Stdout()) checkFifoRead(outr) - checkFifoWrite(tty.Stderr) + checkFifoWrite(tty.io.Stderr()) checkFifoRead(errr) err = outr.Close() @@ -84,8 +85,8 @@ func TestNewTtyIOFifoReopen(t *testing.T) { // Make sure that writing to tty fifo will not get `EPIPE` // when the read side is closed - checkFifoWrite(tty.Stdout) - checkFifoWrite(tty.Stderr) + checkFifoWrite(tty.io.Stdout()) + checkFifoWrite(tty.io.Stderr()) // Reopen the fifo outr = createReadFifo(stdout) @@ -171,7 +172,7 @@ func TestIoCopy(t *testing.T) { defer srcInW.Close() } - tty, err := newTtyIO(ctx, srcStdinPath, dstStdoutPath, dstStderrPath, false) + tty, err := newTtyIO(ctx, "", "", srcStdinPath, dstStdoutPath, dstStderrPath, false) assert.NoError(err) defer tty.close() diff --git a/src/runtime/virtcontainers/device/api/interface.go b/src/runtime/pkg/device/api/interface.go similarity index 90% rename from src/runtime/virtcontainers/device/api/interface.go rename to src/runtime/pkg/device/api/interface.go index 802efe861e..074980f44f 100644 --- a/src/runtime/virtcontainers/device/api/interface.go +++ b/src/runtime/pkg/device/api/interface.go @@ -9,8 +9,7 @@ package api import ( "context" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/sirupsen/logrus" ) @@ -78,10 +77,10 @@ type Device interface { Dereference() uint // Save converts Device to DeviceState - Save() persistapi.DeviceState + Save() config.DeviceState // Load loads DeviceState and converts it to specific device - Load(persistapi.DeviceState) + Load(config.DeviceState) } // DeviceManager can be used to create a new device, this can be used as single @@ -94,5 +93,5 @@ type DeviceManager interface { IsDeviceAttached(string) bool GetDeviceByID(string) Device GetAllDevices() []Device - LoadDevices([]persistapi.DeviceState) + LoadDevices([]config.DeviceState) } diff --git a/src/runtime/virtcontainers/device/api/mockDeviceReceiver.go b/src/runtime/pkg/device/api/mockDeviceReceiver.go similarity index 93% rename from src/runtime/virtcontainers/device/api/mockDeviceReceiver.go rename to src/runtime/pkg/device/api/mockDeviceReceiver.go index d6c23d5d5c..f0bd947f7b 100644 --- a/src/runtime/virtcontainers/device/api/mockDeviceReceiver.go +++ b/src/runtime/pkg/device/api/mockDeviceReceiver.go @@ -8,7 +8,7 @@ package api import ( "context" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ) // MockDeviceReceiver is a fake DeviceReceiver API implementation only used for test diff --git a/src/runtime/virtcontainers/device/config/config.go b/src/runtime/pkg/device/config/config.go similarity index 90% rename from src/runtime/virtcontainers/device/config/config.go rename to src/runtime/pkg/device/config/config.go index 48280092d6..748c5b4e51 100644 --- a/src/runtime/virtcontainers/device/config/config.go +++ b/src/runtime/pkg/device/config/config.go @@ -441,3 +441,45 @@ func getVhostUserDevName(dirname string, majorNum, minorNum uint32) (string, err return "", fmt.Errorf("Required device node (%d:%d) doesn't exist under directory %s", majorNum, minorNum, dirname) } + +// DeviceState is a structure which represents host devices +// plugged to a hypervisor, one Device can be shared among containers in POD +// Refs: virtcontainers/device/drivers/generic.go:GenericDevice +type DeviceState struct { + // DriverOptions is specific options for each device driver + // for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk" + DriverOptions map[string]string + + // VhostUserDeviceAttrs is specific for vhost-user device driver + VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"` + + // BlockDrive is specific for block device driver + BlockDrive *BlockDrive `json:",omitempty"` + + ID string + + // Type is used to specify driver type + // Refs: virtcontainers/device/config/config.go:DeviceType + Type string + + // Type of device: c, b, u or p + // c , u - character(unbuffered) + // p - FIFO + // b - block(buffered) special file + // More info in mknod(1). + DevType string + + // VFIODev is specific VFIO device driver + VFIODevs []*VFIODev `json:",omitempty"` + + RefCount uint + AttachCount uint + + // Major, minor numbers for device. + Major int64 + Minor int64 + + // ColdPlug specifies whether the device must be cold plugged (true) + // or hot plugged (false). + ColdPlug bool +} diff --git a/src/runtime/virtcontainers/device/config/config_test.go b/src/runtime/pkg/device/config/config_test.go similarity index 100% rename from src/runtime/virtcontainers/device/config/config_test.go rename to src/runtime/pkg/device/config/config_test.go diff --git a/src/runtime/virtcontainers/device/config/pmem.go b/src/runtime/pkg/device/config/pmem.go similarity index 100% rename from src/runtime/virtcontainers/device/config/pmem.go rename to src/runtime/pkg/device/config/pmem.go diff --git a/src/runtime/virtcontainers/device/config/pmem_test.go b/src/runtime/pkg/device/config/pmem_test.go similarity index 100% rename from src/runtime/virtcontainers/device/config/pmem_test.go rename to src/runtime/pkg/device/config/pmem_test.go diff --git a/src/runtime/virtcontainers/device/drivers/block.go b/src/runtime/pkg/device/drivers/block.go similarity index 80% rename from src/runtime/virtcontainers/device/drivers/block.go rename to src/runtime/pkg/device/drivers/block.go index faaadf32c9..d2e9644fde 100644 --- a/src/runtime/virtcontainers/device/drivers/block.go +++ b/src/runtime/pkg/device/drivers/block.go @@ -10,9 +10,8 @@ import ( "context" "path/filepath" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -159,51 +158,21 @@ func (device *BlockDevice) GetDeviceInfo() interface{} { } // Save converts Device to DeviceState -func (device *BlockDevice) Save() persistapi.DeviceState { +func (device *BlockDevice) Save() config.DeviceState { ds := device.GenericDevice.Save() ds.Type = string(device.DeviceType()) - drive := device.BlockDrive - if drive != nil { - ds.BlockDrive = &persistapi.BlockDrive{ - File: drive.File, - Format: drive.Format, - ID: drive.ID, - Index: drive.Index, - MmioAddr: drive.MmioAddr, - PCIPath: drive.PCIPath, - SCSIAddr: drive.SCSIAddr, - NvdimmID: drive.NvdimmID, - VirtPath: drive.VirtPath, - DevNo: drive.DevNo, - Pmem: drive.Pmem, - } - } + ds.BlockDrive = device.BlockDrive + return ds } // Load loads DeviceState and converts it to specific device -func (device *BlockDevice) Load(ds persistapi.DeviceState) { +func (device *BlockDevice) Load(ds config.DeviceState) { device.GenericDevice = &GenericDevice{} device.GenericDevice.Load(ds) - bd := ds.BlockDrive - if bd == nil { - return - } - device.BlockDrive = &config.BlockDrive{ - File: bd.File, - Format: bd.Format, - ID: bd.ID, - Index: bd.Index, - MmioAddr: bd.MmioAddr, - PCIPath: bd.PCIPath, - SCSIAddr: bd.SCSIAddr, - NvdimmID: bd.NvdimmID, - VirtPath: bd.VirtPath, - DevNo: bd.DevNo, - Pmem: bd.Pmem, - } + device.BlockDrive = ds.BlockDrive } // It should implement GetAttachCount() and DeviceID() as api.Device implementation diff --git a/src/runtime/virtcontainers/device/drivers/generic.go b/src/runtime/pkg/device/drivers/generic.go similarity index 90% rename from src/runtime/virtcontainers/device/drivers/generic.go rename to src/runtime/pkg/device/drivers/generic.go index fd2bfb671c..38358ed248 100644 --- a/src/runtime/virtcontainers/device/drivers/generic.go +++ b/src/runtime/pkg/device/drivers/generic.go @@ -10,9 +10,8 @@ import ( "context" "fmt" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ) // GenericDevice refers to a device that is neither a VFIO device, block device or VhostUserDevice. @@ -128,8 +127,8 @@ func (device *GenericDevice) bumpAttachCount(attach bool) (skip bool, err error) } // Save converts Device to DeviceState -func (device *GenericDevice) Save() persistapi.DeviceState { - dss := persistapi.DeviceState{ +func (device *GenericDevice) Save() config.DeviceState { + dss := config.DeviceState{ ID: device.ID, Type: string(device.DeviceType()), RefCount: device.RefCount, @@ -148,7 +147,7 @@ func (device *GenericDevice) Save() persistapi.DeviceState { } // Load loads DeviceState and converts it to specific device -func (device *GenericDevice) Load(ds persistapi.DeviceState) { +func (device *GenericDevice) Load(ds config.DeviceState) { device.ID = ds.ID device.RefCount = ds.RefCount device.AttachCount = ds.AttachCount diff --git a/src/runtime/virtcontainers/device/drivers/generic_test.go b/src/runtime/pkg/device/drivers/generic_test.go similarity index 92% rename from src/runtime/virtcontainers/device/drivers/generic_test.go rename to src/runtime/pkg/device/drivers/generic_test.go index e2f6d2e4bd..1dbd505476 100644 --- a/src/runtime/virtcontainers/device/drivers/generic_test.go +++ b/src/runtime/pkg/device/drivers/generic_test.go @@ -8,7 +8,7 @@ package drivers import ( "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/device/drivers/utils.go b/src/runtime/pkg/device/drivers/utils.go similarity index 94% rename from src/runtime/virtcontainers/device/drivers/utils.go rename to src/runtime/pkg/device/drivers/utils.go index aa97747219..25f021eda0 100644 --- a/src/runtime/virtcontainers/device/drivers/utils.go +++ b/src/runtime/pkg/device/drivers/utils.go @@ -12,8 +12,8 @@ import ( "path/filepath" "strings" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/sirupsen/logrus" ) diff --git a/src/runtime/virtcontainers/device/drivers/vfio.go b/src/runtime/pkg/device/drivers/vfio.go similarity index 94% rename from src/runtime/virtcontainers/device/drivers/vfio.go rename to src/runtime/pkg/device/drivers/vfio.go index 8e36ff46ba..58658b0b88 100644 --- a/src/runtime/virtcontainers/device/drivers/vfio.go +++ b/src/runtime/pkg/device/drivers/vfio.go @@ -16,9 +16,8 @@ import ( "github.com/sirupsen/logrus" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -174,26 +173,21 @@ func (device *VFIODevice) GetDeviceInfo() interface{} { } // Save converts Device to DeviceState -func (device *VFIODevice) Save() persistapi.DeviceState { +func (device *VFIODevice) Save() config.DeviceState { ds := device.GenericDevice.Save() ds.Type = string(device.DeviceType()) devs := device.VfioDevs for _, dev := range devs { if dev != nil { - ds.VFIODevs = append(ds.VFIODevs, &persistapi.VFIODev{ - ID: dev.ID, - Type: uint32(dev.Type), - BDF: dev.BDF, - SysfsDev: dev.SysfsDev, - }) + ds.VFIODevs = append(ds.VFIODevs, dev) } } return ds } // Load loads DeviceState and converts it to specific device -func (device *VFIODevice) Load(ds persistapi.DeviceState) { +func (device *VFIODevice) Load(ds config.DeviceState) { device.GenericDevice = &GenericDevice{} device.GenericDevice.Load(ds) diff --git a/src/runtime/virtcontainers/device/drivers/vfio_test.go b/src/runtime/pkg/device/drivers/vfio_test.go similarity index 91% rename from src/runtime/virtcontainers/device/drivers/vfio_test.go rename to src/runtime/pkg/device/drivers/vfio_test.go index 05118871f5..3c25a64c3c 100644 --- a/src/runtime/virtcontainers/device/drivers/vfio_test.go +++ b/src/runtime/pkg/device/drivers/vfio_test.go @@ -9,7 +9,7 @@ package drivers import ( "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/device/drivers/vhost_user_blk.go b/src/runtime/pkg/device/drivers/vhost_user_blk.go similarity index 82% rename from src/runtime/virtcontainers/device/drivers/vhost_user_blk.go rename to src/runtime/pkg/device/drivers/vhost_user_blk.go index 4a495c9d51..49c66e7117 100644 --- a/src/runtime/virtcontainers/device/drivers/vhost_user_blk.go +++ b/src/runtime/pkg/device/drivers/vhost_user_blk.go @@ -9,9 +9,8 @@ package drivers import ( "context" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" ) @@ -156,40 +155,19 @@ func (device *VhostUserBlkDevice) GetDeviceInfo() interface{} { } // Save converts Device to DeviceState -func (device *VhostUserBlkDevice) Save() persistapi.DeviceState { +func (device *VhostUserBlkDevice) Save() config.DeviceState { ds := device.GenericDevice.Save() ds.Type = string(device.DeviceType()) + ds.VhostUserDev = device.VhostUserDeviceAttrs - vAttr := device.VhostUserDeviceAttrs - if vAttr != nil { - ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ - DevID: vAttr.DevID, - SocketPath: vAttr.SocketPath, - Type: string(vAttr.Type), - PCIPath: vAttr.PCIPath, - Index: vAttr.Index, - } - } return ds } // Load loads DeviceState and converts it to specific device -func (device *VhostUserBlkDevice) Load(ds persistapi.DeviceState) { +func (device *VhostUserBlkDevice) Load(ds config.DeviceState) { device.GenericDevice = &GenericDevice{} device.GenericDevice.Load(ds) - - dev := ds.VhostUserDev - if dev == nil { - return - } - - device.VhostUserDeviceAttrs = &config.VhostUserDeviceAttrs{ - DevID: dev.DevID, - SocketPath: dev.SocketPath, - Type: config.DeviceType(dev.Type), - PCIPath: dev.PCIPath, - Index: dev.Index, - } + device.VhostUserDeviceAttrs = ds.VhostUserDev } // It should implement GetAttachCount() and DeviceID() as api.Device implementation diff --git a/src/runtime/virtcontainers/device/drivers/vhost_user_fs.go b/src/runtime/pkg/device/drivers/vhost_user_fs.go similarity index 89% rename from src/runtime/virtcontainers/device/drivers/vhost_user_fs.go rename to src/runtime/pkg/device/drivers/vhost_user_fs.go index 5724d1db0c..dba9ea79df 100644 --- a/src/runtime/virtcontainers/device/drivers/vhost_user_fs.go +++ b/src/runtime/pkg/device/drivers/vhost_user_fs.go @@ -9,8 +9,8 @@ import ( "context" "encoding/hex" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) diff --git a/src/runtime/virtcontainers/device/drivers/vhost_user_net.go b/src/runtime/pkg/device/drivers/vhost_user_net.go similarity index 70% rename from src/runtime/virtcontainers/device/drivers/vhost_user_net.go rename to src/runtime/pkg/device/drivers/vhost_user_net.go index 7b70e97db2..39c38751e2 100644 --- a/src/runtime/virtcontainers/device/drivers/vhost_user_net.go +++ b/src/runtime/pkg/device/drivers/vhost_user_net.go @@ -10,16 +10,15 @@ import ( "context" "encoding/hex" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) // VhostUserNetDevice is a network vhost-user based device type VhostUserNetDevice struct { *GenericDevice - config.VhostUserDeviceAttrs + *config.VhostUserDeviceAttrs } // @@ -71,38 +70,25 @@ func (device *VhostUserNetDevice) DeviceType() config.DeviceType { // GetDeviceInfo returns device information used for creating func (device *VhostUserNetDevice) GetDeviceInfo() interface{} { device.Type = device.DeviceType() - return &device.VhostUserDeviceAttrs + return device.VhostUserDeviceAttrs } // Save converts Device to DeviceState -func (device *VhostUserNetDevice) Save() persistapi.DeviceState { +func (device *VhostUserNetDevice) Save() config.DeviceState { ds := device.GenericDevice.Save() ds.Type = string(device.DeviceType()) - ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ - DevID: device.DevID, - SocketPath: device.SocketPath, - Type: string(device.Type), - MacAddress: device.MacAddress, - } + + ds.VhostUserDev = device.VhostUserDeviceAttrs + return ds } // Load loads DeviceState and converts it to specific device -func (device *VhostUserNetDevice) Load(ds persistapi.DeviceState) { +func (device *VhostUserNetDevice) Load(ds config.DeviceState) { device.GenericDevice = &GenericDevice{} device.GenericDevice.Load(ds) - dev := ds.VhostUserDev - if dev == nil { - return - } - - device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{ - DevID: dev.DevID, - SocketPath: dev.SocketPath, - Type: config.DeviceType(dev.Type), - MacAddress: dev.MacAddress, - } + device.VhostUserDeviceAttrs = ds.VhostUserDev } // It should implement GetAttachCount() and DeviceID() as api.Device implementation diff --git a/src/runtime/virtcontainers/device/drivers/vhost_user_scsi.go b/src/runtime/pkg/device/drivers/vhost_user_scsi.go similarity index 70% rename from src/runtime/virtcontainers/device/drivers/vhost_user_scsi.go rename to src/runtime/pkg/device/drivers/vhost_user_scsi.go index 13b90cdf08..bb9fe97227 100644 --- a/src/runtime/virtcontainers/device/drivers/vhost_user_scsi.go +++ b/src/runtime/pkg/device/drivers/vhost_user_scsi.go @@ -10,16 +10,15 @@ import ( "context" "encoding/hex" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) // VhostUserSCSIDevice is a SCSI vhost-user based device type VhostUserSCSIDevice struct { *GenericDevice - config.VhostUserDeviceAttrs + *config.VhostUserDeviceAttrs } // @@ -71,38 +70,24 @@ func (device *VhostUserSCSIDevice) DeviceType() config.DeviceType { // GetDeviceInfo returns device information used for creating func (device *VhostUserSCSIDevice) GetDeviceInfo() interface{} { device.Type = device.DeviceType() - return &device.VhostUserDeviceAttrs + return device.VhostUserDeviceAttrs } // Save converts Device to DeviceState -func (device *VhostUserSCSIDevice) Save() persistapi.DeviceState { +func (device *VhostUserSCSIDevice) Save() config.DeviceState { ds := device.GenericDevice.Save() ds.Type = string(device.DeviceType()) - ds.VhostUserDev = &persistapi.VhostUserDeviceAttrs{ - DevID: device.DevID, - SocketPath: device.SocketPath, - Type: string(device.Type), - MacAddress: device.MacAddress, - } + ds.VhostUserDev = device.VhostUserDeviceAttrs + return ds } // Load loads DeviceState and converts it to specific device -func (device *VhostUserSCSIDevice) Load(ds persistapi.DeviceState) { +func (device *VhostUserSCSIDevice) Load(ds config.DeviceState) { device.GenericDevice = &GenericDevice{} device.GenericDevice.Load(ds) - dev := ds.VhostUserDev - if dev == nil { - return - } - - device.VhostUserDeviceAttrs = config.VhostUserDeviceAttrs{ - DevID: dev.DevID, - SocketPath: dev.SocketPath, - Type: config.DeviceType(dev.Type), - MacAddress: dev.MacAddress, - } + device.VhostUserDeviceAttrs = ds.VhostUserDev } // It should implement GetAttachCount() and DeviceID() as api.Device implementation diff --git a/src/runtime/virtcontainers/device/manager/manager.go b/src/runtime/pkg/device/manager/manager.go similarity index 94% rename from src/runtime/virtcontainers/device/manager/manager.go rename to src/runtime/pkg/device/manager/manager.go index 9a5bba2604..eed9e39f1e 100644 --- a/src/runtime/virtcontainers/device/manager/manager.go +++ b/src/runtime/pkg/device/manager/manager.go @@ -14,10 +14,9 @@ import ( "github.com/sirupsen/logrus" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" - persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -242,7 +241,7 @@ func (dm *deviceManager) IsDeviceAttached(id string) bool { } // LoadDevices load devices from persist state -func (dm *deviceManager) LoadDevices(devStates []persistapi.DeviceState) { +func (dm *deviceManager) LoadDevices(devStates []config.DeviceState) { dm.Lock() defer dm.Unlock() diff --git a/src/runtime/virtcontainers/device/manager/manager_linux_test.go b/src/runtime/pkg/device/manager/manager_linux_test.go similarity index 90% rename from src/runtime/virtcontainers/device/manager/manager_linux_test.go rename to src/runtime/pkg/device/manager/manager_linux_test.go index abb7532911..fd790f093e 100644 --- a/src/runtime/virtcontainers/device/manager/manager_linux_test.go +++ b/src/runtime/pkg/device/manager/manager_linux_test.go @@ -13,10 +13,10 @@ import ( "path/filepath" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" "github.com/stretchr/testify/assert" "golang.org/x/sys/unix" diff --git a/src/runtime/virtcontainers/device/manager/manager_test.go b/src/runtime/pkg/device/manager/manager_test.go similarity index 96% rename from src/runtime/virtcontainers/device/manager/manager_test.go rename to src/runtime/pkg/device/manager/manager_test.go index ea20012a92..1070e6b8e2 100644 --- a/src/runtime/virtcontainers/device/manager/manager_test.go +++ b/src/runtime/pkg/device/manager/manager_test.go @@ -13,9 +13,9 @@ import ( "strconv" "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/device/manager/utils.go b/src/runtime/pkg/device/manager/utils.go similarity index 92% rename from src/runtime/virtcontainers/device/manager/utils.go rename to src/runtime/pkg/device/manager/utils.go index 61488ef9fd..17d14741c1 100644 --- a/src/runtime/virtcontainers/device/manager/utils.go +++ b/src/runtime/pkg/device/manager/utils.go @@ -10,7 +10,7 @@ import ( "path/filepath" "strings" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ) const ( diff --git a/src/runtime/virtcontainers/device/manager/utils_test.go b/src/runtime/pkg/device/manager/utils_test.go similarity index 95% rename from src/runtime/virtcontainers/device/manager/utils_test.go rename to src/runtime/pkg/device/manager/utils_test.go index ec518ce7ad..273283823f 100644 --- a/src/runtime/virtcontainers/device/manager/utils_test.go +++ b/src/runtime/pkg/device/manager/utils_test.go @@ -9,7 +9,7 @@ package manager import ( "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index c883d9a8de..544fb7cb6f 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -16,12 +16,12 @@ import ( "strings" "github.com/BurntSushi/toml" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/oci" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index 76be6d0243..0081c84de0 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -27,7 +27,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" dockershimAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim" diff --git a/src/runtime/pkg/oci/utils_test.go b/src/runtime/pkg/oci/utils_test.go index 4f778d5699..2ddd42d111 100644 --- a/src/runtime/pkg/oci/utils_test.go +++ b/src/runtime/pkg/oci/utils_test.go @@ -21,8 +21,8 @@ import ( "github.com/stretchr/testify/assert" "golang.org/x/sys/unix" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" dockerAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations/dockershim" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" diff --git a/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/read_closer.go b/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/read_closer.go new file mode 100644 index 0000000000..fbc30a6f73 --- /dev/null +++ b/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/read_closer.go @@ -0,0 +1,57 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ioutil + +import "io" + +// writeCloseInformer wraps a reader with a close function. +type wrapReadCloser struct { + reader *io.PipeReader + writer *io.PipeWriter +} + +// NewWrapReadCloser creates a wrapReadCloser from a reader. +// NOTE(random-liu): To avoid goroutine leakage, the reader passed in +// must be eventually closed by the caller. +func NewWrapReadCloser(r io.Reader) io.ReadCloser { + pr, pw := io.Pipe() + go func() { + _, _ = io.Copy(pw, r) + pr.Close() + pw.Close() + }() + return &wrapReadCloser{ + reader: pr, + writer: pw, + } +} + +// Read reads up to len(p) bytes into p. +func (w *wrapReadCloser) Read(p []byte) (int, error) { + n, err := w.reader.Read(p) + if err == io.ErrClosedPipe { + return n, io.EOF + } + return n, err +} + +// Close closes read closer. +func (w *wrapReadCloser) Close() error { + w.reader.Close() + w.writer.Close() + return nil +} diff --git a/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/write_closer.go b/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/write_closer.go new file mode 100644 index 0000000000..c816c514ad --- /dev/null +++ b/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/write_closer.go @@ -0,0 +1,102 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ioutil + +import ( + "io" + "sync" +) + +// writeCloseInformer wraps passed in write closer with a close channel. +// Caller could wait on the close channel for the write closer to be +// closed. +type writeCloseInformer struct { + close chan struct{} + wc io.WriteCloser +} + +// NewWriteCloseInformer creates the writeCloseInformer from a write closer. +func NewWriteCloseInformer(wc io.WriteCloser) (io.WriteCloser, <-chan struct{}) { + close := make(chan struct{}) + return &writeCloseInformer{ + close: close, + wc: wc, + }, close +} + +// Write passes through the data into the internal write closer. +func (w *writeCloseInformer) Write(p []byte) (int, error) { + return w.wc.Write(p) +} + +// Close closes the internal write closer and inform the close channel. +func (w *writeCloseInformer) Close() error { + err := w.wc.Close() + close(w.close) + return err +} + +// nopWriteCloser wraps passed in writer with a nop close function. +type nopWriteCloser struct { + w io.Writer +} + +// NewNopWriteCloser creates the nopWriteCloser from a writer. +func NewNopWriteCloser(w io.Writer) io.WriteCloser { + return &nopWriteCloser{w: w} +} + +// Write passes through the data into the internal writer. +func (n *nopWriteCloser) Write(p []byte) (int, error) { + return n.w.Write(p) +} + +// Close is a nop close function. +func (n *nopWriteCloser) Close() error { + return nil +} + +// serialWriteCloser wraps a write closer and makes sure all writes +// are done in serial. +// Parallel write won't intersect with each other. Use case: +// 1) Pipe: Write content longer than PIPE_BUF. +// See http://man7.org/linux/man-pages/man7/pipe.7.html +// 2) <3.14 Linux Kernel: write is not atomic +// See http://man7.org/linux/man-pages/man2/write.2.html +type serialWriteCloser struct { + mu sync.Mutex + wc io.WriteCloser +} + +// NewSerialWriteCloser creates a SerialWriteCloser from a write closer. +func NewSerialWriteCloser(wc io.WriteCloser) io.WriteCloser { + return &serialWriteCloser{wc: wc} +} + +// Write writes a group of byte arrays in order atomically. +func (s *serialWriteCloser) Write(data []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.wc.Write(data) +} + +// Close closes the write closer. +func (s *serialWriteCloser) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.wc.Close() +} diff --git a/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/writer_group.go b/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/writer_group.go new file mode 100644 index 0000000000..0ed550497b --- /dev/null +++ b/src/runtime/vendor/github.com/containerd/containerd/pkg/ioutil/writer_group.go @@ -0,0 +1,105 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package ioutil + +import ( + "errors" + "io" + "sync" +) + +// WriterGroup is a group of writers. Writer could be dynamically +// added and removed. +type WriterGroup struct { + mu sync.Mutex + writers map[string]io.WriteCloser + closed bool +} + +var _ io.Writer = &WriterGroup{} + +// NewWriterGroup creates an empty writer group. +func NewWriterGroup() *WriterGroup { + return &WriterGroup{ + writers: make(map[string]io.WriteCloser), + } +} + +// Add adds a writer into the group. The writer will be closed +// if the writer group is closed. +func (g *WriterGroup) Add(key string, w io.WriteCloser) { + g.mu.Lock() + defer g.mu.Unlock() + if g.closed { + w.Close() + return + } + g.writers[key] = w +} + +// Get gets a writer from the group, returns nil if the writer +// doesn't exist. +func (g *WriterGroup) Get(key string) io.WriteCloser { + g.mu.Lock() + defer g.mu.Unlock() + return g.writers[key] +} + +// Remove removes a writer from the group. +func (g *WriterGroup) Remove(key string) { + g.mu.Lock() + defer g.mu.Unlock() + w, ok := g.writers[key] + if !ok { + return + } + w.Close() + delete(g.writers, key) +} + +// Write writes data into each writer. If a writer returns error, +// it will be closed and removed from the writer group. It returns +// error if writer group is empty. +func (g *WriterGroup) Write(p []byte) (int, error) { + g.mu.Lock() + defer g.mu.Unlock() + for k, w := range g.writers { + n, err := w.Write(p) + if err == nil && len(p) == n { + continue + } + // The writer is closed or in bad state, remove it. + w.Close() + delete(g.writers, k) + } + if len(g.writers) == 0 { + return 0, errors.New("writer group is empty") + } + return len(p), nil +} + +// Close closes the writer group. Write will return error after +// closed. +func (g *WriterGroup) Close() { + g.mu.Lock() + defer g.mu.Unlock() + for _, w := range g.writers { + w.Close() + } + g.writers = nil + g.closed = true +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/LICENSE b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/bytes.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/bytes.go new file mode 100644 index 0000000000..7788a1fbe1 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/bytes.go @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bytes + +import ( + "encoding/binary" + "unsafe" +) + +// Raw returns just the bytes without any assumptions about layout +type Raw interface { + Raw() *[]byte +} + +// Reader used to read various data sizes in the byte array +type Reader interface { + Read8(pos int) uint8 + Read16(pos int) uint16 + Read32(pos int) uint32 + Read64(pos int) uint64 + Len() int +} + +// Writer used to write various sizes of data in the byte array +type Writer interface { + Write8(pos int, value uint8) + Write16(pos int, value uint16) + Write32(pos int, value uint32) + Write64(pos int, value uint64) + Len() int +} + +// Bytes object for manipulating arbitrary byte arrays +type Bytes interface { + Raw + Reader + Writer + Slice(offset int, size int) Bytes + LittleEndian() Bytes + BigEndian() Bytes +} + +var nativeByteOrder binary.ByteOrder + +func init() { + buf := [2]byte{} + *(*uint16)(unsafe.Pointer(&buf[0])) = uint16(0x00FF) + + switch buf { + case [2]byte{0xFF, 0x00}: + nativeByteOrder = binary.LittleEndian + case [2]byte{0x00, 0xFF}: + nativeByteOrder = binary.BigEndian + default: + panic("Unable to infer byte order") + } +} + +// New raw bytearray +func New(data *[]byte) Bytes { + return (*native)(data) +} + +// NewLittleEndian little endian ordering of bytes +func NewLittleEndian(data *[]byte) Bytes { + if nativeByteOrder == binary.LittleEndian { + return (*native)(data) + } + + return (*swapbo)(data) +} + +// NewBigEndian big endian ordering of bytes +func NewBigEndian(data *[]byte) Bytes { + if nativeByteOrder == binary.BigEndian { + return (*native)(data) + } + + return (*swapbo)(data) +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/native.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/native.go new file mode 100644 index 0000000000..3c79e6890c --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/native.go @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bytes + +import ( + "unsafe" +) + +type native []byte + +var _ Bytes = (*native)(nil) + +func (b *native) Read8(pos int) uint8 { + return (*b)[pos] +} + +func (b *native) Read16(pos int) uint16 { + return *(*uint16)(unsafe.Pointer(&((*b)[pos]))) +} + +func (b *native) Read32(pos int) uint32 { + return *(*uint32)(unsafe.Pointer(&((*b)[pos]))) +} + +func (b *native) Read64(pos int) uint64 { + return *(*uint64)(unsafe.Pointer(&((*b)[pos]))) +} + +func (b *native) Write8(pos int, value uint8) { + (*b)[pos] = value +} + +func (b *native) Write16(pos int, value uint16) { + *(*uint16)(unsafe.Pointer(&((*b)[pos]))) = value +} + +func (b *native) Write32(pos int, value uint32) { + *(*uint32)(unsafe.Pointer(&((*b)[pos]))) = value +} + +func (b *native) Write64(pos int, value uint64) { + *(*uint64)(unsafe.Pointer(&((*b)[pos]))) = value +} + +func (b *native) Slice(offset int, size int) Bytes { + nb := (*b)[offset : offset+size] + return &nb +} + +func (b *native) LittleEndian() Bytes { + return NewLittleEndian((*[]byte)(b)) +} + +func (b *native) BigEndian() Bytes { + return NewBigEndian((*[]byte)(b)) +} + +func (b *native) Raw() *[]byte { + return (*[]byte)(b) +} + +func (b *native) Len() int { + return len(*b) +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/swapbo.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/swapbo.go new file mode 100644 index 0000000000..278c67daf4 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes/swapbo.go @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package bytes + +import ( + "unsafe" +) + +type swapbo []byte + +var _ Bytes = (*swapbo)(nil) + +func (b *swapbo) Read8(pos int) uint8 { + return (*b)[pos] +} + +func (b *swapbo) Read16(pos int) uint16 { + buf := [2]byte{} + buf[0] = (*b)[pos+1] + buf[1] = (*b)[pos+0] + return *(*uint16)(unsafe.Pointer(&buf[0])) +} + +func (b *swapbo) Read32(pos int) uint32 { + buf := [4]byte{} + buf[0] = (*b)[pos+3] + buf[1] = (*b)[pos+2] + buf[2] = (*b)[pos+1] + buf[3] = (*b)[pos+0] + return *(*uint32)(unsafe.Pointer(&buf[0])) +} + +func (b *swapbo) Read64(pos int) uint64 { + buf := [8]byte{} + buf[0] = (*b)[pos+7] + buf[1] = (*b)[pos+6] + buf[2] = (*b)[pos+5] + buf[3] = (*b)[pos+4] + buf[4] = (*b)[pos+3] + buf[5] = (*b)[pos+2] + buf[6] = (*b)[pos+1] + buf[7] = (*b)[pos+0] + return *(*uint64)(unsafe.Pointer(&buf[0])) +} + +func (b *swapbo) Write8(pos int, value uint8) { + (*b)[pos] = value +} + +func (b *swapbo) Write16(pos int, value uint16) { + buf := [2]byte{} + *(*uint16)(unsafe.Pointer(&buf[0])) = value + (*b)[pos+0] = buf[1] + (*b)[pos+1] = buf[0] +} + +func (b *swapbo) Write32(pos int, value uint32) { + buf := [4]byte{} + *(*uint32)(unsafe.Pointer(&buf[0])) = value + (*b)[pos+0] = buf[3] + (*b)[pos+1] = buf[2] + (*b)[pos+2] = buf[1] + (*b)[pos+3] = buf[0] +} + +func (b *swapbo) Write64(pos int, value uint64) { + buf := [8]byte{} + *(*uint64)(unsafe.Pointer(&buf[0])) = value + (*b)[pos+0] = buf[7] + (*b)[pos+1] = buf[6] + (*b)[pos+2] = buf[5] + (*b)[pos+3] = buf[4] + (*b)[pos+4] = buf[3] + (*b)[pos+5] = buf[2] + (*b)[pos+6] = buf[1] + (*b)[pos+7] = buf[0] +} + +func (b *swapbo) Slice(offset int, size int) Bytes { + nb := (*b)[offset : offset+size] + return &nb +} + +func (b *swapbo) LittleEndian() Bytes { + return NewLittleEndian((*[]byte)(b)) +} + +func (b *swapbo) BigEndian() Bytes { + return NewBigEndian((*[]byte)(b)) +} + +func (b *swapbo) Raw() *[]byte { + return (*[]byte)(b) +} + +func (b *swapbo) Len() int { + return len(*b) +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/config.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/config.go new file mode 100644 index 0000000000..7cd2920b75 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/config.go @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nvpci + +import ( + "fmt" + "io/ioutil" + + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes" +) + +const ( + // PCICfgSpaceStandardSize represents the size in bytes of the standard config space + PCICfgSpaceStandardSize = 256 + // PCICfgSpaceExtendedSize represents the size in bytes of the extended config space + PCICfgSpaceExtendedSize = 4096 + // PCICapabilityListPointer represents offset for the capability list pointer + PCICapabilityListPointer = 0x34 + // PCIStatusCapabilityList represents the status register bit which indicates capability list support + PCIStatusCapabilityList = 0x10 + // PCIStatusBytePosition represents the position of the status register + PCIStatusBytePosition = 0x06 +) + +// ConfigSpace PCI configuration space (standard extended) file path +type ConfigSpace struct { + Path string +} + +// ConfigSpaceIO Interface for reading and writing raw and preconfigured values +type ConfigSpaceIO interface { + bytes.Bytes + GetVendorID() uint16 + GetDeviceID() uint16 + GetPCICapabilities() (*PCICapabilities, error) +} + +type configSpaceIO struct { + bytes.Bytes +} + +// PCIStandardCapability standard PCI config space +type PCIStandardCapability struct { + bytes.Bytes +} + +// PCIExtendedCapability extended PCI config space +type PCIExtendedCapability struct { + bytes.Bytes + Version uint8 +} + +// PCICapabilities combines the standard and extended config space +type PCICapabilities struct { + Standard map[uint8]*PCIStandardCapability + Extended map[uint16]*PCIExtendedCapability +} + +func (cs *ConfigSpace) Read() (ConfigSpaceIO, error) { + config, err := ioutil.ReadFile(cs.Path) + if err != nil { + return nil, fmt.Errorf("failed to open file: %v", err) + } + return &configSpaceIO{bytes.New(&config)}, nil +} + +func (cs *configSpaceIO) GetVendorID() uint16 { + return cs.Read16(0) +} + +func (cs *configSpaceIO) GetDeviceID() uint16 { + return cs.Read16(2) +} + +func (cs *configSpaceIO) GetPCICapabilities() (*PCICapabilities, error) { + caps := &PCICapabilities{ + make(map[uint8]*PCIStandardCapability), + make(map[uint16]*PCIExtendedCapability), + } + + support := cs.Read8(PCIStatusBytePosition) & PCIStatusCapabilityList + if support == 0 { + return nil, fmt.Errorf("pci device does not support capability list") + } + + soffset := cs.Read8(PCICapabilityListPointer) + if int(soffset) >= cs.Len() { + return nil, fmt.Errorf("capability list pointer out of bounds") + } + + for soffset != 0 { + if soffset == 0xff { + return nil, fmt.Errorf("config space broken") + } + if int(soffset) >= PCICfgSpaceStandardSize { + return nil, fmt.Errorf("standard capability list pointer out of bounds") + } + data := cs.Read32(int(soffset)) + id := uint8(data & 0xff) + caps.Standard[id] = &PCIStandardCapability{ + cs.Slice(int(soffset), cs.Len()-int(soffset)), + } + soffset = uint8((data >> 8) & 0xff) + } + + if cs.Len() <= PCICfgSpaceStandardSize { + return caps, nil + } + + eoffset := uint16(PCICfgSpaceStandardSize) + for eoffset != 0 { + if eoffset == 0xffff { + return nil, fmt.Errorf("config space broken") + } + if int(eoffset) >= PCICfgSpaceExtendedSize { + return nil, fmt.Errorf("extended capability list pointer out of bounds") + } + data := cs.Read32(int(eoffset)) + id := uint16(data & 0xffff) + version := uint8((data >> 16) & 0xf) + caps.Extended[id] = &PCIExtendedCapability{ + cs.Slice(int(eoffset), cs.Len()-int(eoffset)), + version, + } + eoffset = uint16((data >> 4) & 0xffc) + } + + return caps, nil +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mmio.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mmio.go new file mode 100644 index 0000000000..602486ead6 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mmio.go @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mmio + +import ( + "fmt" + "os" + "syscall" + "unsafe" + + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes" +) + +// Mmio memory map a region +type Mmio interface { + bytes.Raw + bytes.Reader + bytes.Writer + Sync() error + Close() error + Slice(offset int, size int) Mmio + LittleEndian() Mmio + BigEndian() Mmio +} + +type mmio struct { + bytes.Bytes +} + +func open(path string, offset int, size int, flags int) (Mmio, error) { + var mmapFlags int + switch flags { + case os.O_RDONLY: + mmapFlags = syscall.PROT_READ + case os.O_RDWR: + mmapFlags = syscall.PROT_READ | syscall.PROT_WRITE + default: + return nil, fmt.Errorf("invalid flags: %v", flags) + } + + file, err := os.OpenFile(path, flags, 0) + if err != nil { + return nil, fmt.Errorf("failed to open file: %v", err) + } + defer file.Close() + + fi, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("failed to get file info: %v", err) + } + + if size > int(fi.Size()) { + return nil, fmt.Errorf("requested size larger than file size") + } + + if size < 0 { + size = int(fi.Size()) + } + + mmap, err := syscall.Mmap( + int(file.Fd()), + int64(offset), + size, + mmapFlags, + syscall.MAP_SHARED) + if err != nil { + return nil, fmt.Errorf("failed to mmap file: %v", err) + } + + return &mmio{bytes.New(&mmap)}, nil +} + +// OpenRO open region readonly +func OpenRO(path string, offset int, size int) (Mmio, error) { + return open(path, offset, size, os.O_RDONLY) +} + +// OpenRW open region read write +func OpenRW(path string, offset int, size int) (Mmio, error) { + return open(path, offset, size, os.O_RDWR) +} + +func (m *mmio) Slice(offset int, size int) Mmio { + return &mmio{m.Bytes.Slice(offset, size)} +} + +func (m *mmio) LittleEndian() Mmio { + return &mmio{m.Bytes.LittleEndian()} +} + +func (m *mmio) BigEndian() Mmio { + return &mmio{m.Bytes.BigEndian()} +} + +func (m *mmio) Close() error { + err := syscall.Munmap(*m.Bytes.Raw()) + if err != nil { + return fmt.Errorf("failed to munmap file: %v", err) + } + return nil +} + +func (m *mmio) Sync() error { + _, _, errno := syscall.Syscall( + syscall.SYS_MSYNC, + uintptr(unsafe.Pointer(&(*m.Bytes.Raw())[0])), + uintptr(m.Len()), + uintptr(syscall.MS_SYNC|syscall.MS_INVALIDATE)) + if errno != 0 { + return fmt.Errorf("failed to msync file: %v", errno) + } + return nil +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mock.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mock.go new file mode 100644 index 0000000000..42a86b157e --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio/mock.go @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package mmio + +import ( + "fmt" + + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes" +) + +type mockMmio struct { + mmio + source *[]byte + offset int + rw bool +} + +func mockOpen(source *[]byte, offset int, size int, rw bool) (Mmio, error) { + if size < 0 { + size = len(*source) - offset + } + if (offset + size) > len(*source) { + return nil, fmt.Errorf("offset+size out of range") + } + + data := append([]byte{}, (*source)[offset:offset+size]...) + + m := &mockMmio{} + m.Bytes = bytes.New(&data).LittleEndian() + m.source = source + m.offset = offset + m.rw = rw + + return m, nil +} + +// MockOpenRO open read only +func MockOpenRO(source *[]byte, offset int, size int) (Mmio, error) { + return mockOpen(source, offset, size, false) +} + +// MockOpenRW open read write +func MockOpenRW(source *[]byte, offset int, size int) (Mmio, error) { + return mockOpen(source, offset, size, true) +} + +func (m *mockMmio) Close() error { + m = &mockMmio{} + return nil +} + +func (m *mockMmio) Sync() error { + if !m.rw { + return fmt.Errorf("opened read-only") + } + for i := range *m.Bytes.Raw() { + (*m.source)[m.offset+i] = (*m.Bytes.Raw())[i] + } + return nil +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mock.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mock.go new file mode 100644 index 0000000000..5c13ae1861 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mock.go @@ -0,0 +1,141 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nvpci + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes" +) + +// MockNvpci mock pci device +type MockNvpci struct { + *nvpci +} + +var _ Interface = (*MockNvpci)(nil) + +// NewMockNvpci create new mock PCI and remove old devices +func NewMockNvpci() (mock *MockNvpci, rerr error) { + rootDir, err := ioutil.TempDir("", "") + if err != nil { + return nil, err + } + defer func() { + if rerr != nil { + os.RemoveAll(rootDir) + } + }() + + mock = &MockNvpci{ + NewFrom(rootDir).(*nvpci), + } + + return mock, nil +} + +// Cleanup remove the mocked PCI devices root folder +func (m *MockNvpci) Cleanup() { + os.RemoveAll(m.pciDevicesRoot) +} + +// AddMockA100 Create an A100 like GPU mock device +func (m *MockNvpci) AddMockA100(address string, numaNode int) error { + deviceDir := filepath.Join(m.pciDevicesRoot, address) + err := os.MkdirAll(deviceDir, 0755) + if err != nil { + return err + } + + vendor, err := os.Create(filepath.Join(deviceDir, "vendor")) + if err != nil { + return err + } + _, err = vendor.WriteString(fmt.Sprintf("0x%x", PCINvidiaVendorID)) + if err != nil { + return err + } + + class, err := os.Create(filepath.Join(deviceDir, "class")) + if err != nil { + return err + } + _, err = class.WriteString(fmt.Sprintf("0x%x", PCI3dControllerClass)) + if err != nil { + return err + } + + device, err := os.Create(filepath.Join(deviceDir, "device")) + if err != nil { + return err + } + _, err = device.WriteString("0x20bf") + if err != nil { + return err + } + + numa, err := os.Create(filepath.Join(deviceDir, "numa_node")) + if err != nil { + return err + } + _, err = numa.WriteString(fmt.Sprintf("%v", numaNode)) + if err != nil { + return err + } + + config, err := os.Create(filepath.Join(deviceDir, "config")) + if err != nil { + return err + } + _data := make([]byte, PCICfgSpaceStandardSize) + data := bytes.New(&_data) + data.Write16(0, PCINvidiaVendorID) + data.Write16(2, uint16(0x20bf)) + data.Write8(PCIStatusBytePosition, PCIStatusCapabilityList) + _, err = config.Write(*data.Raw()) + if err != nil { + return err + } + + bar0 := []uint64{0x00000000c2000000, 0x00000000c2ffffff, 0x0000000000040200} + resource, err := os.Create(filepath.Join(deviceDir, "resource")) + if err != nil { + return err + } + _, err = resource.WriteString(fmt.Sprintf("0x%x 0x%x 0x%x", bar0[0], bar0[1], bar0[2])) + if err != nil { + return err + } + + pmcID := uint32(0x170000a1) + resource0, err := os.Create(filepath.Join(deviceDir, "resource0")) + if err != nil { + return err + } + _data = make([]byte, bar0[1]-bar0[0]+1) + data = bytes.New(&_data).LittleEndian() + data.Write32(0, pmcID) + _, err = resource0.Write(*data.Raw()) + if err != nil { + return err + } + + return nil +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/nvpci.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/nvpci.go new file mode 100644 index 0000000000..61a8bd302d --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/nvpci.go @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nvpci + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "sort" + "strconv" + "strings" +) + +const ( + // PCIDevicesRoot represents base path for all pci devices under sysfs + PCIDevicesRoot = "/sys/bus/pci/devices" + // PCINvidiaVendorID represents PCI vendor id for NVIDIA + PCINvidiaVendorID uint16 = 0x10de + // PCIVgaControllerClass represents the PCI class for VGA Controllers + PCIVgaControllerClass uint32 = 0x030000 + // PCI3dControllerClass represents the PCI class for 3D Graphics accellerators + PCI3dControllerClass uint32 = 0x030200 + // PCINvSwitchClass represents the PCI class for NVSwitches + PCINvSwitchClass uint32 = 0x068000 +) + +// Interface allows us to get a list of all NVIDIA PCI devices +type Interface interface { + GetAllDevices() ([]*NvidiaPCIDevice, error) + Get3DControllers() ([]*NvidiaPCIDevice, error) + GetVGAControllers() ([]*NvidiaPCIDevice, error) + GetNVSwitches() ([]*NvidiaPCIDevice, error) + GetGPUs() ([]*NvidiaPCIDevice, error) +} + +// MemoryResources a more human readable handle +type MemoryResources map[int]*MemoryResource + +// ResourceInterface exposes some higher level functions of resources +type ResourceInterface interface { + GetTotalAddressableMemory(bool) (uint64, uint64) +} + +type nvpci struct { + pciDevicesRoot string +} + +var _ Interface = (*nvpci)(nil) +var _ ResourceInterface = (*MemoryResources)(nil) + +// NvidiaPCIDevice represents a PCI device for an NVIDIA product +type NvidiaPCIDevice struct { + Path string + Address string + Vendor uint16 + Class uint32 + Device uint16 + NumaNode int + Config *ConfigSpace + Resources MemoryResources +} + +// IsVGAController if class == 0x300 +func (d *NvidiaPCIDevice) IsVGAController() bool { + return d.Class == PCIVgaControllerClass +} + +// Is3DController if class == 0x302 +func (d *NvidiaPCIDevice) Is3DController() bool { + return d.Class == PCI3dControllerClass +} + +// IsNVSwitch if classe == 0x068 +func (d *NvidiaPCIDevice) IsNVSwitch() bool { + return d.Class == PCINvSwitchClass +} + +// IsGPU either VGA for older cards or 3D for newer +func (d *NvidiaPCIDevice) IsGPU() bool { + return d.IsVGAController() || d.Is3DController() +} + +// IsResetAvailable some devices can be reset without rebooting, +// check if applicable +func (d *NvidiaPCIDevice) IsResetAvailable() bool { + _, err := os.Stat(path.Join(d.Path, "reset")) + return err == nil +} + +// Reset perform a reset to apply a new configuration at HW level +func (d *NvidiaPCIDevice) Reset() error { + err := ioutil.WriteFile(path.Join(d.Path, "reset"), []byte("1"), 0) + if err != nil { + return fmt.Errorf("unable to write to reset file: %v", err) + } + return nil +} + +// New interface that allows us to get a list of all NVIDIA PCI devices +func New() Interface { + return &nvpci{PCIDevicesRoot} +} + +// NewFrom interface allows us to get a list of all NVIDIA PCI devices at a specific root directory +func NewFrom(root string) Interface { + return &nvpci{root} +} + +// GetAllDevices returns all Nvidia PCI devices on the system +func (p *nvpci) GetAllDevices() ([]*NvidiaPCIDevice, error) { + deviceDirs, err := ioutil.ReadDir(p.pciDevicesRoot) + if err != nil { + return nil, fmt.Errorf("unable to read PCI bus devices: %v", err) + } + + var nvdevices []*NvidiaPCIDevice + for _, deviceDir := range deviceDirs { + devicePath := path.Join(p.pciDevicesRoot, deviceDir.Name()) + nvdevice, err := NewDevice(devicePath) + if err != nil { + return nil, fmt.Errorf("error constructing NVIDIA PCI device %s: %v", deviceDir.Name(), err) + } + if nvdevice == nil { + continue + } + nvdevices = append(nvdevices, nvdevice) + } + + addressToID := func(address string) uint64 { + address = strings.ReplaceAll(address, ":", "") + address = strings.ReplaceAll(address, ".", "") + id, _ := strconv.ParseUint(address, 16, 64) + return id + } + + sort.Slice(nvdevices, func(i, j int) bool { + return addressToID(nvdevices[i].Address) < addressToID(nvdevices[j].Address) + }) + + return nvdevices, nil +} + +// NewDevice constructs an NvidiaPCIDevice +func NewDevice(devicePath string) (*NvidiaPCIDevice, error) { + address := path.Base(devicePath) + + vendor, err := ioutil.ReadFile(path.Join(devicePath, "vendor")) + if err != nil { + return nil, fmt.Errorf("unable to read PCI device vendor id for %s: %v", address, err) + } + vendorStr := strings.TrimSpace(string(vendor)) + vendorID, err := strconv.ParseUint(vendorStr, 0, 16) + if err != nil { + return nil, fmt.Errorf("unable to convert vendor string to uint16: %v", vendorStr) + } + + if uint16(vendorID) != PCINvidiaVendorID { + return nil, nil + } + + class, err := ioutil.ReadFile(path.Join(devicePath, "class")) + if err != nil { + return nil, fmt.Errorf("unable to read PCI device class for %s: %v", address, err) + } + classStr := strings.TrimSpace(string(class)) + classID, err := strconv.ParseUint(classStr, 0, 32) + if err != nil { + return nil, fmt.Errorf("unable to convert class string to uint32: %v", classStr) + } + + device, err := ioutil.ReadFile(path.Join(devicePath, "device")) + if err != nil { + return nil, fmt.Errorf("unable to read PCI device id for %s: %v", address, err) + } + deviceStr := strings.TrimSpace(string(device)) + deviceID, err := strconv.ParseUint(deviceStr, 0, 16) + if err != nil { + return nil, fmt.Errorf("unable to convert device string to uint16: %v", deviceStr) + } + + numa, err := ioutil.ReadFile(path.Join(devicePath, "numa_node")) + if err != nil { + return nil, fmt.Errorf("unable to read PCI NUMA node for %s: %v", address, err) + } + numaStr := strings.TrimSpace(string(numa)) + numaNode, err := strconv.ParseInt(numaStr, 0, 64) + if err != nil { + return nil, fmt.Errorf("unable to convert NUMA node string to int64: %v", numaNode) + } + + config := &ConfigSpace{ + Path: path.Join(devicePath, "config"), + } + + resource, err := ioutil.ReadFile(path.Join(devicePath, "resource")) + if err != nil { + return nil, fmt.Errorf("unable to read PCI resource file for %s: %v", address, err) + } + + resources := make(map[int]*MemoryResource) + for i, line := range strings.Split(strings.TrimSpace(string(resource)), "\n") { + values := strings.Split(line, " ") + if len(values) != 3 { + return nil, fmt.Errorf("more than 3 entries in line '%d' of resource file", i) + } + + start, _ := strconv.ParseUint(values[0], 0, 64) + end, _ := strconv.ParseUint(values[1], 0, 64) + flags, _ := strconv.ParseUint(values[2], 0, 64) + + if (end - start) != 0 { + resources[i] = &MemoryResource{ + uintptr(start), + uintptr(end), + flags, + fmt.Sprintf("%s/resource%d", devicePath, i), + } + } + } + + nvdevice := &NvidiaPCIDevice{ + Path: devicePath, + Address: address, + Vendor: uint16(vendorID), + Class: uint32(classID), + Device: uint16(deviceID), + NumaNode: int(numaNode), + Config: config, + Resources: resources, + } + + return nvdevice, nil +} + +// Get3DControllers returns all NVIDIA 3D Controller PCI devices on the system +func (p *nvpci) Get3DControllers() ([]*NvidiaPCIDevice, error) { + devices, err := p.GetAllDevices() + if err != nil { + return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err) + } + + var filtered []*NvidiaPCIDevice + for _, d := range devices { + if d.Is3DController() { + filtered = append(filtered, d) + } + } + + return filtered, nil +} + +// GetVGAControllers returns all NVIDIA VGA Controller PCI devices on the system +func (p *nvpci) GetVGAControllers() ([]*NvidiaPCIDevice, error) { + devices, err := p.GetAllDevices() + if err != nil { + return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err) + } + + var filtered []*NvidiaPCIDevice + for _, d := range devices { + if d.IsVGAController() { + filtered = append(filtered, d) + } + } + + return filtered, nil +} + +// GetNVSwitches returns all NVIDIA NVSwitch PCI devices on the system +func (p *nvpci) GetNVSwitches() ([]*NvidiaPCIDevice, error) { + devices, err := p.GetAllDevices() + if err != nil { + return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err) + } + + var filtered []*NvidiaPCIDevice + for _, d := range devices { + if d.IsNVSwitch() { + filtered = append(filtered, d) + } + } + + return filtered, nil +} + +// GetGPUs returns all NVIDIA GPU devices on the system +func (p *nvpci) GetGPUs() ([]*NvidiaPCIDevice, error) { + devices, err := p.GetAllDevices() + if err != nil { + return nil, fmt.Errorf("error getting all NVIDIA devices: %v", err) + } + + var filtered []*NvidiaPCIDevice + for _, d := range devices { + if d.IsGPU() { + filtered = append(filtered, d) + } + } + + return filtered, nil +} diff --git a/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/resources.go b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/resources.go new file mode 100644 index 0000000000..02a0430f38 --- /dev/null +++ b/src/runtime/vendor/gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/resources.go @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nvpci + +import ( + "fmt" + "sort" + + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio" +) + +const ( + pmcEndianRegister = 0x4 + pmcLittleEndian = 0x0 + pmcBigEndian = 0x01000001 +) + +// MemoryResource represents a mmio region +type MemoryResource struct { + Start uintptr + End uintptr + Flags uint64 + Path string +} + +// OpenRW read write mmio region +func (mr *MemoryResource) OpenRW() (mmio.Mmio, error) { + rw, err := mmio.OpenRW(mr.Path, 0, int(mr.End-mr.Start+1)) + if err != nil { + return nil, fmt.Errorf("failed to open file for mmio: %v", err) + } + switch rw.Read32(pmcEndianRegister) { + case pmcBigEndian: + return rw.BigEndian(), nil + case pmcLittleEndian: + return rw.LittleEndian(), nil + } + return nil, fmt.Errorf("unknown endianness for mmio: %v", err) +} + +// OpenRO read only mmio region +func (mr *MemoryResource) OpenRO() (mmio.Mmio, error) { + ro, err := mmio.OpenRO(mr.Path, 0, int(mr.End-mr.Start+1)) + if err != nil { + return nil, fmt.Errorf("failed to open file for mmio: %v", err) + } + switch ro.Read32(pmcEndianRegister) { + case pmcBigEndian: + return ro.BigEndian(), nil + case pmcLittleEndian: + return ro.LittleEndian(), nil + } + return nil, fmt.Errorf("unknown endianness for mmio: %v", err) +} + +// From Bit Twiddling Hacks, great resource for all low level bit manipulations +func calcNextPowerOf2(n uint64) uint64 { + n-- + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n |= n >> 32 + n++ + + return n +} + +// GetTotalAddressableMemory will accumulate the 32bit and 64bit memory windows +// of each BAR and round the value if needed to the next power of 2; first +// return value is the accumulated 32bit addresable memory size the second one +// is the accumulated 64bit addressable memory size in bytes. These values are +// needed to configure virtualized environments. +func (mrs MemoryResources) GetTotalAddressableMemory(roundUp bool) (uint64, uint64) { + const pciIOVNumBAR = 6 + const pciBaseAddressMemTypeMask = 0x06 + const pciBaseAddressMemType32 = 0x00 /* 32 bit address */ + const pciBaseAddressMemType64 = 0x04 /* 64 bit address */ + + // We need to sort the resources so the first 6 entries are the BARs + // How a map is represented in memory is not guaranteed, it is not an + // array. Keys do not have an order. + keys := make([]int, 0, len(mrs)) + for k := range mrs { + keys = append(keys, k) + } + sort.Ints(keys) + + numBAR := 0 + memSize32bit := uint64(0) + memSize64bit := uint64(0) + + for _, key := range keys { + // The PCIe spec only defines 5 BARs per device, we're + // discarding everything after the 5th entry of the resources + // file, see lspci.c + if key >= pciIOVNumBAR || numBAR == pciIOVNumBAR { + break + } + numBAR = numBAR + 1 + + region := mrs[key] + + flags := region.Flags & pciBaseAddressMemTypeMask + memType32bit := flags == pciBaseAddressMemType32 + memType64bit := flags == pciBaseAddressMemType64 + + memSize := (region.End - region.Start) + 1 + + if memType32bit { + memSize32bit = memSize32bit + uint64(memSize) + } + if memType64bit { + memSize64bit = memSize64bit + uint64(memSize) + } + + } + + if roundUp { + memSize32bit = calcNextPowerOf2(memSize32bit) + memSize64bit = calcNextPowerOf2(memSize64bit) + } + + return memSize32bit, memSize64bit +} diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index d2552ff907..b0152f32fe 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -81,6 +81,7 @@ github.com/containerd/containerd/mount github.com/containerd/containerd/namespaces github.com/containerd/containerd/pkg/cri/annotations github.com/containerd/containerd/pkg/dialer +github.com/containerd/containerd/pkg/ioutil github.com/containerd/containerd/pkg/runtimeoptions/v1 github.com/containerd/containerd/pkg/shutdown github.com/containerd/containerd/pkg/ttrpcutil @@ -305,6 +306,11 @@ github.com/vishvananda/netlink/nl # github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f ## explicit github.com/vishvananda/netns +# gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20220601114329-47893b162965 +## explicit +gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci +gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes +gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio # go.opencensus.io v0.23.0 go.opencensus.io go.opencensus.io/internal diff --git a/src/runtime/virtcontainers/acrn.go b/src/runtime/virtcontainers/acrn.go index 38f42a207b..f6da05ec58 100644 --- a/src/runtime/virtcontainers/acrn.go +++ b/src/runtime/virtcontainers/acrn.go @@ -22,10 +22,10 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" diff --git a/src/runtime/virtcontainers/acrn_arch_base.go b/src/runtime/virtcontainers/acrn_arch_base.go index ab547b6bd3..f84019ee61 100644 --- a/src/runtime/virtcontainers/acrn_arch_base.go +++ b/src/runtime/virtcontainers/acrn_arch_base.go @@ -16,7 +16,7 @@ import ( "os/exec" "strings" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" ) diff --git a/src/runtime/virtcontainers/acrn_arch_base_test.go b/src/runtime/virtcontainers/acrn_arch_base_test.go index 19d7858c59..3d168f56f9 100644 --- a/src/runtime/virtcontainers/acrn_arch_base_test.go +++ b/src/runtime/virtcontainers/acrn_arch_base_test.go @@ -12,7 +12,7 @@ import ( "path/filepath" "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" diff --git a/src/runtime/virtcontainers/acrn_test.go b/src/runtime/virtcontainers/acrn_test.go index b92f35a2fa..f0e8c7ce2c 100644 --- a/src/runtime/virtcontainers/acrn_test.go +++ b/src/runtime/virtcontainers/acrn_test.go @@ -14,7 +14,7 @@ import ( "path/filepath" "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" diff --git a/src/runtime/virtcontainers/api.go b/src/runtime/virtcontainers/api.go index 4b85851cf6..437c926a7e 100644 --- a/src/runtime/virtcontainers/api.go +++ b/src/runtime/virtcontainers/api.go @@ -9,10 +9,10 @@ import ( "context" "runtime" + deviceApi "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + deviceConfig "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" - deviceApi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - deviceConfig "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/compatoci" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" diff --git a/src/runtime/virtcontainers/clh.go b/src/runtime/virtcontainers/clh.go index ce8f8fa577..6984b73be5 100644 --- a/src/runtime/virtcontainers/clh.go +++ b/src/runtime/virtcontainers/clh.go @@ -32,9 +32,9 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -753,6 +753,11 @@ func (clh *cloudHypervisor) hotplugAddBlockDevice(drive *config.BlockDrive) erro clhDisk.Readonly = &drive.ReadOnly clhDisk.VhostUser = func(b bool) *bool { return &b }(false) + queues := int32(clh.config.NumVCPUs) + queueSize := int32(1024) + clhDisk.NumQueues = &queues + clhDisk.QueueSize = &queueSize + diskRateLimiterConfig := clh.getDiskRateLimiterConfig() if diskRateLimiterConfig != nil { clhDisk.SetRateLimiterConfig(*diskRateLimiterConfig) diff --git a/src/runtime/virtcontainers/clh_test.go b/src/runtime/virtcontainers/clh_test.go index 302d945f00..bb4a04925b 100644 --- a/src/runtime/virtcontainers/clh_test.go +++ b/src/runtime/virtcontainers/clh_test.go @@ -18,7 +18,7 @@ import ( "strings" "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" chclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/cloud-hypervisor/client" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 4ba260e00b..6dd9b6dc36 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -16,10 +16,10 @@ import ( "syscall" "time" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" vcAnnotations "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/annotations" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" diff --git a/src/runtime/virtcontainers/container_test.go b/src/runtime/virtcontainers/container_test.go index f6fce13e62..7a165ce9fc 100644 --- a/src/runtime/virtcontainers/container_test.go +++ b/src/runtime/virtcontainers/container_test.go @@ -14,11 +14,11 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" diff --git a/src/runtime/virtcontainers/fc.go b/src/runtime/virtcontainers/fc.go index b792c90acb..25012d3f3d 100644 --- a/src/runtime/virtcontainers/fc.go +++ b/src/runtime/virtcontainers/fc.go @@ -24,9 +24,9 @@ import ( "syscall" "time" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client" models "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/firecracker/client/models" diff --git a/src/runtime/virtcontainers/fs_share_linux.go b/src/runtime/virtcontainers/fs_share_linux.go index cec3d154eb..2e80bfcc96 100644 --- a/src/runtime/virtcontainers/fs_share_linux.go +++ b/src/runtime/virtcontainers/fs_share_linux.go @@ -19,8 +19,8 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index 20d7d61038..119c4667c3 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -16,9 +16,9 @@ import ( "github.com/pkg/errors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" diff --git a/src/runtime/virtcontainers/interfaces.go b/src/runtime/virtcontainers/interfaces.go index 272059fcab..7bb3993c3a 100644 --- a/src/runtime/virtcontainers/interfaces.go +++ b/src/runtime/virtcontainers/interfaces.go @@ -10,8 +10,8 @@ import ( "io" "syscall" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 38fcae111a..073f360d2b 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -19,14 +19,14 @@ import ( "time" "github.com/docker/go-units" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" + "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" kataclient "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/client" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index c24934b682..27a6837e9c 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -23,11 +23,11 @@ import ( "github.com/stretchr/testify/assert" "code.cloudfoundry.org/bytefmt" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" pb "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols/grpc" diff --git a/src/runtime/virtcontainers/persist.go b/src/runtime/virtcontainers/persist.go index 199c647ae2..59c6dda15f 100644 --- a/src/runtime/virtcontainers/persist.go +++ b/src/runtime/virtcontainers/persist.go @@ -8,8 +8,9 @@ package virtcontainers import ( "errors" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + devconfig "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" @@ -65,7 +66,7 @@ func (s *Sandbox) dumpHypervisor(ss *persistapi.SandboxState) { ss.HypervisorState.BlockIndexMap = s.state.BlockIndexMap } -func deviceToDeviceState(devices []api.Device) (dss []persistapi.DeviceState) { +func deviceToDeviceState(devices []api.Device) (dss []devconfig.DeviceState) { for _, dev := range devices { dss = append(dss, dev.Save()) } @@ -324,7 +325,7 @@ func (s *Sandbox) loadAgent(as persistapi.AgentState) { } } -func (s *Sandbox) loadDevices(devStates []persistapi.DeviceState) { +func (s *Sandbox) loadDevices(devStates []devconfig.DeviceState) { s.devManager.LoadDevices(devStates) } diff --git a/src/runtime/virtcontainers/persist/api/device.go b/src/runtime/virtcontainers/persist/api/device.go deleted file mode 100644 index dd61efbfcf..0000000000 --- a/src/runtime/virtcontainers/persist/api/device.go +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) 2016 Intel Corporation -// Copyright (c) 2019 Huawei Corporation -// -// SPDX-License-Identifier: Apache-2.0 -// - -package persistapi - -import vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - -// ============= sandbox level resources ============= - -// BlockDrive represents a block storage drive which may be used in case the storage -// driver has an underlying block storage device. -type BlockDrive struct { - // File is the path to the disk-image/device which will be used with this drive - File string - - // Format of the drive - Format string - - // ID is used to identify this drive in the hypervisor options. - ID string - - // MmioAddr is used to identify the slot at which the drive is attached (order?). - MmioAddr string - - // SCSI Address of the block device, in case the device is attached using SCSI driver - // SCSI address is in the format SCSI-Id:LUN - SCSIAddr string - - // NvdimmID is the nvdimm id inside the VM - NvdimmID string - - // VirtPath at which the device appears inside the VM, outside of the container mount namespace - VirtPath string - - // DevNo - DevNo string - - // PCIPath is the PCI path used to identify the slot at which the drive is attached. - PCIPath vcTypes.PciPath - - // Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index - Index int - - // Pmem enabled persistent memory. Use File as backing file - // for a nvdimm device in the guest. - Pmem bool -} - -// VFIODev represents a VFIO drive used for hotplugging -type VFIODev struct { - // ID is used to identify this drive in the hypervisor options. - ID string - - // BDF (Bus:Device.Function) of the PCI address - BDF string - - // Sysfsdev of VFIO mediated device - SysfsDev string - - // Type of VFIO device - Type uint32 -} - -// VhostUserDeviceAttrs represents data shared by most vhost-user devices -type VhostUserDeviceAttrs struct { - DevID string - SocketPath string - Type string - - // MacAddress is only meaningful for vhost user net device - MacAddress string - - // PCIPath is the PCI path used to identify the slot at which the drive is attached. - // It is only meaningful for vhost user block devices - PCIPath vcTypes.PciPath - - // Block index of the device if assigned - Index int -} - -// DeviceState is sandbox level resource which represents host devices -// plugged to hypervisor, one Device can be shared among containers in POD -// Refs: virtcontainers/device/drivers/generic.go:GenericDevice -type DeviceState struct { - // DriverOptions is specific options for each device driver - // for example, for BlockDevice, we can set DriverOptions["block-driver"]="virtio-blk" - DriverOptions map[string]string - - // VhostUserDeviceAttrs is specific for vhost-user device driver - VhostUserDev *VhostUserDeviceAttrs `json:",omitempty"` - - // BlockDrive is specific for block device driver - BlockDrive *BlockDrive `json:",omitempty"` - - ID string - - // Type is used to specify driver type - // Refs: virtcontainers/device/config/config.go:DeviceType - Type string - - // Type of device: c, b, u or p - // c , u - character(unbuffered) - // p - FIFO - // b - block(buffered) special file - // More info in mknod(1). - DevType string - - // VFIODev is specific VFIO device driver - VFIODevs []*VFIODev `json:",omitempty"` - - RefCount uint - AttachCount uint - - // Major, minor numbers for device. - Major int64 - Minor int64 - - // ColdPlug specifies whether the device must be cold plugged (true) - // or hot plugged (false). - ColdPlug bool -} diff --git a/src/runtime/virtcontainers/persist/api/sandbox.go b/src/runtime/virtcontainers/persist/api/sandbox.go index 09196637ca..665edd2804 100644 --- a/src/runtime/virtcontainers/persist/api/sandbox.go +++ b/src/runtime/virtcontainers/persist/api/sandbox.go @@ -7,6 +7,7 @@ package persistapi import ( + dev "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" ) @@ -26,7 +27,7 @@ type SandboxState struct { CgroupPaths map[string]string // Devices plugged to sandbox(hypervisor) - Devices []DeviceState + Devices []dev.DeviceState // State is sandbox running status State string diff --git a/src/runtime/virtcontainers/persist_test.go b/src/runtime/virtcontainers/persist_test.go index b6c096a5d9..96b947e5b9 100644 --- a/src/runtime/virtcontainers/persist_test.go +++ b/src/runtime/virtcontainers/persist_test.go @@ -10,8 +10,8 @@ import ( "os" "testing" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" diff --git a/src/runtime/virtcontainers/physical_endpoint.go b/src/runtime/virtcontainers/physical_endpoint.go index 71f67da8fd..ea64ba926b 100644 --- a/src/runtime/virtcontainers/physical_endpoint.go +++ b/src/runtime/virtcontainers/physical_endpoint.go @@ -15,9 +15,9 @@ import ( "path/filepath" "strings" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/safchain/ethtool" diff --git a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go index 12710fb733..f0560dd5e2 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/sandbox.go +++ b/src/runtime/virtcontainers/pkg/vcmock/sandbox.go @@ -11,9 +11,9 @@ import ( "io" "syscall" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" diff --git a/src/runtime/virtcontainers/pkg/vcmock/types.go b/src/runtime/virtcontainers/pkg/vcmock/types.go index 1125090a1b..05a0a98592 100644 --- a/src/runtime/virtcontainers/pkg/vcmock/types.go +++ b/src/runtime/virtcontainers/pkg/vcmock/types.go @@ -10,9 +10,9 @@ import ( "io" "syscall" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" pbTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/pkg/agent/protocols" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" specs "github.com/opencontainers/runtime-spec/specs-go" diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index b56ffda051..c8e48953f6 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -33,12 +33,12 @@ import ( "github.com/sirupsen/logrus" "golang.org/x/sys/unix" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" hv "github.com/kata-containers/kata-containers/src/runtime/pkg/hypervisors" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" pkgUtils "github.com/kata-containers/kata-containers/src/runtime/pkg/utils" "github.com/kata-containers/kata-containers/src/runtime/pkg/uuid" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -681,8 +681,10 @@ func (q *qemu) CreateVM(ctx context.Context, id string, network Network, hypervi // Add PCIe Root Port devices to hypervisor // The pcie.0 bus do not support hot-plug, but PCIe device can be hot-plugged into PCIe Root Port. // For more details, please see https://github.com/qemu/qemu/blob/master/docs/pcie.txt + memSize32bit, memSize64bit := q.arch.getBARsMaxAddressableMemory() + if hypervisorConfig.PCIeRootPort > 0 { - qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort) + qemuConfig.Devices = q.arch.appendPCIeRootPortDevice(qemuConfig.Devices, hypervisorConfig.PCIeRootPort, memSize32bit, memSize64bit) } q.qemuConfig = qemuConfig @@ -1362,7 +1364,9 @@ func (q *qemu) hotplugAddBlockDevice(ctx context.Context, drive *config.BlockDri return err } - if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, 0, true, defaultDisableModern); err != nil { + queues := int(q.config.NumVCPUs) + + if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, queues, true, defaultDisableModern); err != nil { return err } case q.config.BlockDeviceDriver == config.VirtioBlockCCW: @@ -2373,7 +2377,7 @@ func genericMemoryTopology(memoryMb, hostMemoryMb uint64, slots uint8, memoryOff } // genericAppendPCIeRootPort appends to devices the given pcie-root-port -func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string) []govmmQemu.Device { +func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machineType string, memSize32bit uint64, memSize64bit uint64) []govmmQemu.Device { var ( bus string chassis string @@ -2399,6 +2403,8 @@ func genericAppendPCIeRootPort(devices []govmmQemu.Device, number uint32, machin Slot: strconv.FormatUint(uint64(i), 10), Multifunction: multiFunction, Addr: addr, + MemReserve: fmt.Sprintf("%dB", memSize32bit), + Pref64Reserve: fmt.Sprintf("%dB", memSize64bit), }, ) } diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index a2873a2160..1d7d76bfae 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -18,8 +18,9 @@ import ( "strings" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" + "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" ) @@ -138,7 +139,7 @@ type qemuArch interface { setIgnoreSharedMemoryMigrationCaps(context.Context, *govmmQemu.QMP) error // appendPCIeRootPortDevice appends a pcie-root-port device to pcie.0 bus - appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device + appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32, memSize32bit uint64, memSize64bit uint64) []govmmQemu.Device // append vIOMMU device appendIOMMU(devices []govmmQemu.Device) ([]govmmQemu.Device, error) @@ -151,6 +152,10 @@ type qemuArch interface { // a firmware, returns a string containing the path to the firmware that should // be used with the -bios option, ommit -bios option if the path is empty. appendProtectionDevice(devices []govmmQemu.Device, firmware, firmwareVolume string) ([]govmmQemu.Device, string, error) + + // scans the PCIe space and returns the biggest BAR sizes for 32-bit + // and 64-bit addressable memory + getBARsMaxAddressableMemory() (uint64, uint64) } type qemuArchBase struct { @@ -787,8 +792,39 @@ func (q *qemuArchBase) addBridge(b types.Bridge) { } // appendPCIeRootPortDevice appends to devices the given pcie-root-port -func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32) []govmmQemu.Device { - return genericAppendPCIeRootPort(devices, number, q.qemuMachine.Type) +func (q *qemuArchBase) appendPCIeRootPortDevice(devices []govmmQemu.Device, number uint32, memSize32bit uint64, memSize64bit uint64) []govmmQemu.Device { + return genericAppendPCIeRootPort(devices, number, q.qemuMachine.Type, memSize32bit, memSize64bit) +} + +func (q *qemuArchBase) getBARsMaxAddressableMemory() (uint64, uint64) { + + pci := nvpci.New() + devs, _ := pci.GetAllDevices() + + // Since we do not know which devices are going to be hotplugged, + // we're going to use the GPU with the biggest BARs to initialize the + // root port, this should work for all other devices as well. + // defaults are 2MB for both, if no suitable devices found + max32bit := uint64(2 * 1024 * 1024) + max64bit := uint64(2 * 1024 * 1024) + + for _, dev := range devs { + if !dev.IsGPU() { + continue + } + memSize32bit, memSize64bit := dev.Resources.GetTotalAddressableMemory(true) + if max32bit < memSize32bit { + max32bit = memSize32bit + } + if max64bit < memSize64bit { + max64bit = memSize64bit + } + } + // The actual 32bit is most of the time a power of 2 but we need some + // buffer so double that to leave space for other IO functions. + // The 64bit size is not a power of 2 and hence is already rounded up + // to the higher value. + return max32bit * 2, max64bit } // appendIOMMU appends a virtual IOMMU device diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index 57c97f773b..41d3bbd430 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -19,7 +19,7 @@ import ( govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/stretchr/testify/assert" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/pkg/errors" diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index f0946698f7..46a1af4ad5 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -13,8 +13,8 @@ import ( "fmt" "time" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" "github.com/sirupsen/logrus" diff --git a/src/runtime/virtcontainers/qemu_s390x_test.go b/src/runtime/virtcontainers/qemu_s390x_test.go index 3d0c393a69..ada3cefc6e 100644 --- a/src/runtime/virtcontainers/qemu_s390x_test.go +++ b/src/runtime/virtcontainers/qemu_s390x_test.go @@ -16,7 +16,7 @@ import ( govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" "github.com/stretchr/testify/assert" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" ) func newTestQemu(assert *assert.Assertions, machineType string) qemuArch { diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index 0c27240be6..b50d73a917 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -15,9 +15,9 @@ import ( "path/filepath" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm" govmmQemu "github.com/kata-containers/kata-containers/src/runtime/pkg/govmm/qemu" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index d768793b74..8fe40ece82 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -25,12 +25,12 @@ import ( "github.com/sirupsen/logrus" "github.com/vishvananda/netlink" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/api" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" + deviceManager "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" resCtrl "github.com/kata-containers/kata-containers/src/runtime/pkg/resourcecontrol" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/api" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" - deviceManager "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/image" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" diff --git a/src/runtime/virtcontainers/sandbox_test.go b/src/runtime/virtcontainers/sandbox_test.go index 31bec93818..331094ee39 100644 --- a/src/runtime/virtcontainers/sandbox_test.go +++ b/src/runtime/virtcontainers/sandbox_test.go @@ -16,10 +16,10 @@ import ( "syscall" "testing" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/drivers" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/manager" ktu "github.com/kata-containers/kata-containers/src/runtime/pkg/katatestutils" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/drivers" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/fs" diff --git a/src/runtime/virtcontainers/vhostuser_endpoint.go b/src/runtime/virtcontainers/vhostuser_endpoint.go index ecdfa8a5ef..c27f1159d7 100644 --- a/src/runtime/virtcontainers/vhostuser_endpoint.go +++ b/src/runtime/virtcontainers/vhostuser_endpoint.go @@ -14,7 +14,7 @@ import ( "fmt" "os" - "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" + "github.com/kata-containers/kata-containers/src/runtime/pkg/device/config" persistapi "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist/api" vcTypes "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/utils" diff --git a/tools/packaging/kernel/README.md b/tools/packaging/kernel/README.md index 4f89121e03..cb4840260e 100644 --- a/tools/packaging/kernel/README.md +++ b/tools/packaging/kernel/README.md @@ -47,7 +47,7 @@ Options: -g : GPU vendor, intel or nvidia. -h : Display this help. -k : Path to kernel to build. - -p : Path to a directory with patches to apply to kernel. + -p : Path to a directory with patches to apply to kernel, only patches in top-level directory are applied. -t : Hypervisor_target. -v : Kernel version to use if kernel path not provided. ``` @@ -76,7 +76,7 @@ $ ./build-kernel.sh setup The script `./build-kernel.sh` tries to apply the patches from `${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/patches/` when it sets up a kernel. If you want to add a source modification, add a patch on this -directory. +directory. Patches present in the top-level directory are applied, with subdirectories being ignored. The script also adds a kernel config file from `${GOPATH}/src/github.com/kata-containers/kata-containers/tools/packaging/kernel/configs/` to `.config` diff --git a/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf b/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf index 214c469b58..a363ec6b6e 100644 --- a/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf +++ b/tools/packaging/kernel/configs/fragments/x86_64/tdx/tdx.conf @@ -1,5 +1,6 @@ # Intel Trust Domain Extensions (Intel TDX) +CONFIG_EFI=y CONFIG_EFI_STUB=y CONFIG_X86_INTEL_MEMORY_PROTECTION_KEYS=y CONFIG_INTEL_TDX_GUEST=y diff --git a/tools/packaging/kernel/kata_config_version b/tools/packaging/kernel/kata_config_version index cd5b025272..c67f579c9a 100644 --- a/tools/packaging/kernel/kata_config_version +++ b/tools/packaging/kernel/kata_config_version @@ -1 +1 @@ -92 +93 diff --git a/tools/packaging/scripts/apply_patches.sh b/tools/packaging/scripts/apply_patches.sh index e5092779f5..c5b8c748d7 100755 --- a/tools/packaging/scripts/apply_patches.sh +++ b/tools/packaging/scripts/apply_patches.sh @@ -36,7 +36,7 @@ fi echo "INFO: Apply patches from $patches_dir" if [ -d "$patches_dir" ]; then - patches=($(find "$patches_dir" -name '*.patch'|sort -t- -k1,1n)) + patches=($(find "$patches_dir" -maxdepth 1 -name '*.patch'|sort -t- -k1,1n)) echo "INFO: Found ${#patches[@]} patches" for patch in ${patches[@]}; do echo "INFO: Apply $patch" diff --git a/tools/packaging/static-build/scripts/kata-configure-docker.sh b/tools/packaging/static-build/scripts/kata-configure-docker.sh deleted file mode 100644 index 72fbc28e01..0000000000 --- a/tools/packaging/static-build/scripts/kata-configure-docker.sh +++ /dev/null @@ -1,146 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) 2019 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -# Description: Script to configure Docker for the static -# version of Kata Containers. - -[ -z "${DEBUG}" ] || set -x -set -o errexit -set -o nounset -set -o pipefail - -docker_config_dir="/etc/docker" -docker_config_file="${docker_config_file:-${docker_config_dir}/daemon.json}" - -# The static version of Kata Containers is entirely contained within -# this directory. -readonly static_base_dir="/opt/kata" - -# Path to runtime in static archive file -readonly runtime_path="${static_base_dir}/bin/kata-runtime" - -die() -{ - local msg="$*" - echo >&2 "ERROR: $msg" - exit 1 -} - -info() -{ - local msg="$*" - echo >&2 "INFO: $msg" -} - -configure_docker() -{ - local file="$1" - [ -z "$file" ] && die "need file" - - mkdir -p "${docker_config_dir}" - - if [ -e "$docker_config_file" ] - then - local today=$(date '+%Y-%m-%d') - local backup="${docker_config_file}.${today}" - - info "Backing up original Docker config file '$docker_config_file' to '$backup'" - - sudo cp "${docker_config_file}" "${docker_config_file}.${today}" - else - # Create a minimal valid JSON document - echo "{}" > "${docker_config_file}" - fi - - local config_files=$(tar tvf "$file" |\ - grep "/configuration-.*\.toml" |\ - grep -v -- '->' |\ - awk '{print $NF}' |\ - sed 's/^\.//g' || true) - - [ -z "$config_files" ] && die "cannot find any configuration files in '$file'" - - local config - local -a runtimes - - for config in $(echo "$config_files" | tr '\n' ' ') - do - local runtime - runtime=$(echo "$config" |\ - awk -F \/ '{print $NF}' |\ - sed -e 's/configuration/kata/g' -e 's/\.toml//g') - - runtimes+=("$runtime") - - local result - result=$(cat "$docker_config_file" |\ - jq \ - --arg config "$config" \ - --arg runtime "$runtime" \ - --arg runtime_path "$runtime_path" \ - '.runtimes[$runtime] = {path: $runtime_path, "runtimeArgs": ["--config", $config]}') - - echo "$result" > "$docker_config_file" - done - - info "Validating $docker_config_file" - - jq -S . "$docker_config_file" &>/dev/null - - info "Restarting Docker to apply new configuration" - - $chronic sudo systemctl restart docker - - info "Docker configured for the following additional runtimes: ${runtimes[@]}" -} - -setup() -{ - source "/etc/os-release" || source "/usr/lib/os-release" - - # Used to manipulate $docker_config_file - local pkg="jq" - - case "$ID" in - opensuse*) distro="opensuse" ;; - *) distro="$ID" ;; - esac - - # Use chronic(1) if available - chronic= - command -v chronic && chronic=chronic - - if command -v "$pkg" &>/dev/null - then - return 0 - fi - - info "Cannot find $pkg command so installing package" - - case "$distro" in - centos|rhel) $chronic sudo -E yum -y install "$pkg" ;; - debian|ubuntu) $chronic sudo -E apt-get --no-install-recommends install -y "$pkg" ;; - fedora) $chronic sudo -E dnf -y install "$pkg" ;; - opensuse|sles) $chronic sudo -E zypper -y install "$pkg" ;; - *) die "do not know how to install command $pkg' for distro '$distro'" ;; - esac -} - -main() -{ - local file="$1" - [ -z "$file" ] && die "need full path to Kata Containers static archive file" - - echo "$file" | grep -q "^kata-static-.*\.tar.xz" || die "invalid file: '$file'" - - [ $(id -u) -eq 0 ] || die "must be run as root" - - setup - - configure_docker "$file" -} - -main "$@" diff --git a/versions.yaml b/versions.yaml index 60a357ce0a..2774d6a48c 100644 --- a/versions.yaml +++ b/versions.yaml @@ -151,7 +151,7 @@ assets: kernel: description: "Linux kernel optimised for virtual machines" url: "https://cdn.kernel.org/pub/linux/kernel/v5.x/" - version: "v5.15.26" + version: "v5.15.48" tdx: description: "Linux kernel that supports TDX" url: "https://github.com/intel/tdx/archive/refs/tags"