diff --git a/src/runtime/go.mod b/src/runtime/go.mod index 882cc61085..dc42ad0ddd 100644 --- a/src/runtime/go.mod +++ b/src/runtime/go.mod @@ -32,7 +32,7 @@ require ( github.com/gogo/googleapis v1.4.0 // indirect github.com/gogo/protobuf v1.3.1 github.com/hashicorp/go-multierror v1.0.0 - github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 + github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c diff --git a/src/runtime/go.sum b/src/runtime/go.sum index 5ad94ef100..4716fc89a6 100644 --- a/src/runtime/go.sum +++ b/src/runtime/go.sum @@ -164,12 +164,6 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/intel/govmm v0.0.0-20200724170648-af9e34b91ae9 h1:GSRnVLRNweZV3f8Vo3vtMlPsuzvpj57Gj7Y7TkGVO6U= -github.com/intel/govmm v0.0.0-20200724170648-af9e34b91ae9/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g= -github.com/intel/govmm v0.0.0-20200728135209-6c3315ba8a42 h1:Yu3/MlZl/kKE0Ipgio/KVorMIrjeHTVOG4+9WAddgOQ= -github.com/intel/govmm v0.0.0-20200728135209-6c3315ba8a42/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g= -github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 h1:yltaUdR0Vitnn/FEfy+JWbJ+oGhMAPP/3S7ja9S5yso= -github.com/intel/govmm v0.0.0-20200825065022-6042f6033126/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -179,7 +173,8 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kata-containers/kata-containers v0.0.0-20201013034856-c88820454d08 h1:yk9fzLKb9RmV9xuT5mkJw4owk/K0rX5cusm2ukEEDro= +github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca h1:UdXFthwasAPnmv37gLJUEFsW9FaabYA+mM6FoSi8kzU= +github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index ca857ca97b..754090941c 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -14,7 +14,7 @@ import ( "strings" "github.com/BurntSushi/toml" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental" diff --git a/src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md b/src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md deleted file mode 100644 index 7a822adbe0..0000000000 --- a/src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md +++ /dev/null @@ -1,22 +0,0 @@ -This file is a partial list of contributors to the Virtual Machine -Manager for Go project. To see the full list of contributors, see the -revision history in source control. - -Contributors who wish to be recognized in this file should add -themselves (or their employer, as appropriate). - -- afrosi@de.ibm.com -- archana.m.shinde@intel.com -- caoruidong@huawei.com -- clare.chenhui@huawei.com -- eric.ernst@intel.com -- james.o.hunt@intel.com -- jose.carlos.venegas.munoz@intel.com -- julio.montes@intel.com -- manohar.r.castelino@intel.com -- mark.d.ryan@intel.com -- robert.bradford@intel.com -- sameo@linux.intel.com -- sebastien.boeuf@intel.com -- teawater@hyper.sh -- xinda.zhao@intel.com diff --git a/src/runtime/vendor/github.com/intel/govmm/COPYING b/src/runtime/vendor/github.com/kata-containers/govmm/COPYING similarity index 100% rename from src/runtime/vendor/github.com/intel/govmm/COPYING rename to src/runtime/vendor/github.com/kata-containers/govmm/COPYING diff --git a/src/runtime/vendor/github.com/intel/govmm/qemu/image.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/image.go similarity index 100% rename from src/runtime/vendor/github.com/intel/govmm/qemu/image.go rename to src/runtime/vendor/github.com/kata-containers/govmm/qemu/image.go diff --git a/src/runtime/vendor/github.com/intel/govmm/qemu/qemu.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go similarity index 96% rename from src/runtime/vendor/github.com/intel/govmm/qemu/qemu.go rename to src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go index 9288f59f8a..9cfe39cbd1 100644 --- a/src/runtime/vendor/github.com/intel/govmm/qemu/qemu.go +++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go @@ -135,7 +135,7 @@ const ( func isDimmSupported(config *Config) bool { switch runtime.GOARCH { - case "amd64", "386", "ppc64le": + case "amd64", "386", "ppc64le", "arm64": if config != nil && config.Machine.Type == MachineTypeMicrovm { // microvm does not support NUMA return false @@ -1084,6 +1084,8 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string { deviceParams = append(deviceParams, fmt.Sprintf(",share-rw=on")) } + deviceParams = append(deviceParams, fmt.Sprintf(",serial=%s", blkdev.ID)) + blkParams = append(blkParams, fmt.Sprintf("id=%s", blkdev.ID)) blkParams = append(blkParams, fmt.Sprintf(",file=%s", blkdev.File)) blkParams = append(blkParams, fmt.Sprintf(",aio=%s", blkdev.AIO)) @@ -1118,6 +1120,24 @@ func (blkdev BlockDevice) deviceName(config *Config) string { return string(blkdev.Driver) } +// PVPanicDevice represents a qemu pvpanic device. +type PVPanicDevice struct { + NoShutdown bool +} + +// Valid always returns true for pvpanic device +func (dev PVPanicDevice) Valid() bool { + return true +} + +// QemuParams returns the qemu parameters built out of this serial device. +func (dev PVPanicDevice) QemuParams(config *Config) []string { + if dev.NoShutdown { + return []string{"-device", "pvpanic", "-no-shutdown"} + } + return []string{"-device", "pvpanic"} +} + // VhostUserDevice represents a qemu vhost-user device meant to be passed // in to the guest type VhostUserDevice struct { @@ -2103,6 +2123,56 @@ type Kernel struct { Params string } +// FwCfg allows QEMU to pass entries to the guest +// File and Str are mutually exclusive +type FwCfg struct { + Name string + File string + Str string +} + +// Valid returns true if the FwCfg structure is valid and complete. +func (fwcfg FwCfg) Valid() bool { + if fwcfg.Name == "" { + return false + } + + if fwcfg.File != "" && fwcfg.Str != "" { + return false + } + + if fwcfg.File == "" && fwcfg.Str == "" { + return false + } + + return true +} + +// QemuParams returns the qemu parameters built out of the FwCfg object +func (fwcfg FwCfg) QemuParams(config *Config) []string { + var fwcfgParams []string + var qemuParams []string + + for _, f := range config.FwCfg { + if f.Name != "" { + fwcfgParams = append(fwcfgParams, fmt.Sprintf("name=%s", f.Name)) + + if f.File != "" { + fwcfgParams = append(fwcfgParams, fmt.Sprintf(",file=%s", f.File)) + } + + if f.Str != "" { + fwcfgParams = append(fwcfgParams, fmt.Sprintf(",string=%s", f.Str)) + } + } + + qemuParams = append(qemuParams, "-fw_cfg") + qemuParams = append(qemuParams, strings.Join(fwcfgParams, "")) + } + + return qemuParams +} + // Knobs regroups a set of qemu boolean settings type Knobs struct { // NoUserConfig prevents qemu from loading user config files. @@ -2230,12 +2300,18 @@ type Config struct { // Bios is the -bios parameter Bios string + // PFlash specifies the parallel flash images (-pflash parameter) + PFlash []string + // Incoming controls migration source preparation Incoming Incoming // fds is a list of open file descriptors to be passed to the spawned qemu process fds []*os.File + // FwCfg is the -fw_cfg parameter + FwCfg []FwCfg + IOThreads []IOThread // PidFile is the -pidfile parameter @@ -2417,6 +2493,13 @@ func (config *Config) appendGlobalParam() { } } +func (config *Config) appendPFlashParam() { + for _, p := range config.PFlash { + config.qemuParams = append(config.qemuParams, "-pflash") + config.qemuParams = append(config.qemuParams, p) + } +} + func (config *Config) appendVGA() { if config.VGA != "" { config.qemuParams = append(config.qemuParams, "-vga") @@ -2568,6 +2651,21 @@ func (config *Config) appendLogFile() { } } +func (config *Config) appendFwCfg(logger QMPLog) { + if logger == nil { + logger = qmpNullLogger{} + } + + for _, f := range config.FwCfg { + if !f.Valid() { + logger.Errorf("fw_cfg is not valid: %+v", config.FwCfg) + continue + } + + config.qemuParams = append(config.qemuParams, f.QemuParams(config)...) + } +} + // LaunchQemu can be used to launch a new qemu instance. // // The Config parameter contains a set of qemu parameters and settings. @@ -2587,6 +2685,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) { config.appendDevices() config.appendRTC() config.appendGlobalParam() + config.appendPFlashParam() config.appendVGA() config.appendKnobs() config.appendKernel() @@ -2595,6 +2694,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) { config.appendIncoming() config.appendPidFile() config.appendLogFile() + config.appendFwCfg(logger) if err := config.appendCPUs(); err != nil { return "", err diff --git a/src/runtime/vendor/github.com/intel/govmm/qemu/qmp.go b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go similarity index 97% rename from src/runtime/vendor/github.com/intel/govmm/qemu/qmp.go rename to src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go index 4942727c42..73d2ec4cff 100644 --- a/src/runtime/vendor/github.com/intel/govmm/qemu/qmp.go +++ b/src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go @@ -268,7 +268,7 @@ func (q *QMP) readLoop(fromVMCh chan<- []byte) { for scanner.Scan() { line := scanner.Bytes() if q.cfg.Logger.V(1) { - q.cfg.Logger.Infof("%s", string(line)) + q.cfg.Logger.Infof("read from QMP: %s", string(line)) } // Since []byte channel type transfer slice info(include slice underlying array pointer, len, cap) @@ -773,11 +773,12 @@ func (q *QMP) ExecuteQuit(ctx context.Context) error { return q.executeCommand(ctx, "quit", nil, nil) } -func (q *QMP) blockdevAddBaseArgs(device, blockdevID string) (map[string]interface{}, map[string]interface{}) { +func (q *QMP) blockdevAddBaseArgs(device, blockdevID string, ro bool) (map[string]interface{}, map[string]interface{}) { var args map[string]interface{} blockdevArgs := map[string]interface{}{ - "driver": "raw", + "driver": "raw", + "read-only": ro, "file": map[string]interface{}{ "driver": "file", "filename": device, @@ -801,8 +802,8 @@ func (q *QMP) blockdevAddBaseArgs(device, blockdevID string) (map[string]interfa // path of the device to add, e.g., /dev/rdb0, and blockdevID is an identifier // used to name the device. As this identifier will be passed directly to QMP, // it must obey QMP's naming rules, e,g., it must start with a letter. -func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string) error { - args, _ := q.blockdevAddBaseArgs(device, blockdevID) +func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string, ro bool) error { + args, _ := q.blockdevAddBaseArgs(device, blockdevID, ro) return q.executeCommand(ctx, "blockdev-add", args, nil) } @@ -814,8 +815,8 @@ func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string) // direct denotes whether use of O_DIRECT (bypass the host page cache) // is enabled. noFlush denotes whether flush requests for the device are // ignored. -func (q *QMP) ExecuteBlockdevAddWithCache(ctx context.Context, device, blockdevID string, direct, noFlush bool) error { - args, blockdevArgs := q.blockdevAddBaseArgs(device, blockdevID) +func (q *QMP) ExecuteBlockdevAddWithCache(ctx context.Context, device, blockdevID string, direct, noFlush, ro bool) error { + args, blockdevArgs := q.blockdevAddBaseArgs(device, blockdevID, ro) if q.version.Major < 2 || (q.version.Major == 2 && q.version.Minor < 9) { return fmt.Errorf("versions of qemu (%d.%d) older than 2.9 do not support set cache-related options for block devices", @@ -1639,3 +1640,29 @@ func (q *QMP) ExecQomSet(ctx context.Context, path, property string, value uint6 return q.executeCommand(ctx, "qom-set", args, nil) } + +// ExecQomGet qom-get path property +func (q *QMP) ExecQomGet(ctx context.Context, path, property string) (interface{}, error) { + args := map[string]interface{}{ + "path": path, + "property": property, + } + + response, err := q.executeCommandWithResponse(ctx, "qom-get", args, nil, nil) + if err != nil { + return "", err + } + + return response, nil +} + +// ExecuteDumpGuestMemory dump guest memory to host +func (q *QMP) ExecuteDumpGuestMemory(ctx context.Context, protocol string, paging bool, format string) error { + args := map[string]interface{}{ + "protocol": protocol, + "paging": paging, + "format": format, + } + + return q.executeCommand(ctx, "dump-guest-memory", args, nil) +} diff --git a/src/runtime/vendor/modules.txt b/src/runtime/vendor/modules.txt index 11374b356a..68278cfd1c 100644 --- a/src/runtime/vendor/modules.txt +++ b/src/runtime/vendor/modules.txt @@ -222,9 +222,9 @@ github.com/hashicorp/errwrap # github.com/hashicorp/go-multierror v1.0.0 ## explicit github.com/hashicorp/go-multierror -# github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 +# github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca ## explicit -github.com/intel/govmm/qemu +github.com/kata-containers/govmm/qemu # github.com/konsorten/go-windows-terminal-sequences v1.0.1 github.com/konsorten/go-windows-terminal-sequences # github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 21f18f35d7..719e43f266 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -481,7 +481,7 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s } // mountSharedDirMounts handles bind-mounts by bindmounting to the host shared -// directory which is mounted through 9pfs in the VM. +// directory which is mounted through virtiofs/9pfs in the VM. // It also updates the container mount list with the HostPath info, and store // container mounts to the storage. This way, we will have the HostPath info // available when we will need to unmount those mounts. @@ -504,6 +504,18 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( continue } + // Check if mount is a block device file. If it is, the block device will be attached to the host + // instead of passing this as a shared mount: + if len(m.BlockDeviceID) > 0 { + // Attach this block device, all other devices passed in the config have been attached at this point + if err = c.sandbox.devManager.AttachDevice(m.BlockDeviceID, c.sandbox); err != nil { + return nil, nil, err + } + devicesToDetach = append(devicesToDetach, m.BlockDeviceID) + continue + } + + // For non-block based mounts, we are only interested in bind mounts if m.Type != "bind" { continue } @@ -515,17 +527,6 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( continue } - // Check if mount is a block device file. If it is, the block device will be attached to the host - // instead of passing this as a shared mount. - if len(m.BlockDeviceID) > 0 { - // Attach this block device, all other devices passed in the config have been attached at this point - if err = c.sandbox.devManager.AttachDevice(m.BlockDeviceID, c.sandbox); err != nil { - return nil, nil, err - } - devicesToDetach = append(devicesToDetach, m.BlockDeviceID) - continue - } - // Ignore /dev, directories and all other device files. We handle // only regular files in /dev. It does not make sense to pass the host // device nodes to the guest. @@ -631,6 +632,9 @@ func filterDevices(c *Container, devices []ContainerDevice) (ret []ContainerDevi return } +// Add any mount based block devices to the device manager and save the +// device ID for the particular mount. This'll occur when the mountpoint source +// is a block device. func (c *Container) createBlockDevices() error { if !c.checkBlockDeviceSupport() { c.Logger().Warn("Block device not supported") @@ -639,13 +643,18 @@ func (c *Container) createBlockDevices() error { // iterate all mounts and create block device if it's block based. for i, m := range c.mounts { - if len(m.BlockDeviceID) > 0 || m.Type != "bind" { + if len(m.BlockDeviceID) > 0 { // Non-empty m.BlockDeviceID indicates there's already one device // associated with the mount,so no need to create a new device for it // and we only create block device for bind mount continue } + if m.Type != "bind" { + // We only handle for bind-mounts + continue + } + var stat unix.Stat_t if err := unix.Stat(m.Source, &stat); err != nil { return fmt.Errorf("stat %q failed: %v", m.Source, err) @@ -663,6 +672,7 @@ func (c *Container) createBlockDevices() error { DevType: "b", Major: int64(unix.Major(stat.Rdev)), Minor: int64(unix.Minor(stat.Rdev)), + ReadOnly: m.ReadOnly, } // check whether source can be used as a pmem device } else if di, err = config.PmemDeviceInfo(m.Source, m.Destination); err != nil { @@ -673,7 +683,6 @@ func (c *Container) createBlockDevices() error { if err == nil && di != nil { b, err := c.sandbox.devManager.NewDevice(*di) - if err != nil { // Do not return an error, try to create // devices for other mounts @@ -725,11 +734,12 @@ func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, er return nil, err } - // Go to next step for first created container + // If mounts are block devices, add to devmanager if err := c.createMounts(); err != nil { return nil, err } + // Add container's devices to sandbox's device-manager if err := c.createDevices(contConfig); err != nil { return nil, err } @@ -739,11 +749,7 @@ func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, er func (c *Container) createMounts() error { // Create block devices for newly created container - if err := c.createBlockDevices(); err != nil { - return err - } - - return nil + return c.createBlockDevices() } func (c *Container) createDevices(contConfig *ContainerConfig) error { @@ -813,6 +819,7 @@ func (c *Container) create() (err error) { }() if c.checkBlockDeviceSupport() { + // If the rootfs is backed by a block device, go ahead and hotplug it to the guest if err = c.hotplugDrive(); err != nil { return } @@ -1208,11 +1215,14 @@ func (c *Container) resume() error { return c.setContainerState(types.StateRunning) } +// hotplugDrive will attempt to hotplug the container rootfs if it is backed by a +// block device func (c *Container) hotplugDrive() error { var dev device var err error - // container rootfs is blockdevice backed and isn't mounted + // Check to see if the rootfs is an umounted block device (source) or if the + // mount (target) is backed by a block device: if !c.rootFs.Mounted { dev, err = getDeviceForPath(c.rootFs.Source) // there is no "rootfs" dir on block device backed rootfs @@ -1274,6 +1284,7 @@ func (c *Container) hotplugDrive() error { return c.setStateFstype(fsType) } +// plugDevice will attach the rootfs if blockdevice is supported (this is rootfs specific) func (c *Container) plugDevice(devicePath string) error { var stat unix.Stat_t if err := unix.Stat(devicePath, &stat); err != nil { diff --git a/src/runtime/virtcontainers/device/config/config.go b/src/runtime/virtcontainers/device/config/config.go index 6146e8885d..e59278ee2c 100644 --- a/src/runtime/virtcontainers/device/config/config.go +++ b/src/runtime/virtcontainers/device/config/config.go @@ -114,6 +114,9 @@ type DeviceInfo struct { // for a nvdimm device in the guest. Pmem bool + // If applicable, should this device be considered RO + ReadOnly bool + // ColdPlug specifies whether the device must be cold plugged (true) // or hot plugged (false). ColdPlug bool diff --git a/src/runtime/virtcontainers/device/drivers/block.go b/src/runtime/virtcontainers/device/drivers/block.go index 7c88d937e0..e37e69c551 100644 --- a/src/runtime/virtcontainers/device/drivers/block.go +++ b/src/runtime/virtcontainers/device/drivers/block.go @@ -61,11 +61,12 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) { } drive := &config.BlockDrive{ - File: device.DeviceInfo.HostPath, - Format: "raw", - ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize), - Index: index, - Pmem: device.DeviceInfo.Pmem, + File: device.DeviceInfo.HostPath, + Format: "raw", + ID: utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize), + Index: index, + Pmem: device.DeviceInfo.Pmem, + ReadOnly: device.DeviceInfo.ReadOnly, } if fs, ok := device.DeviceInfo.DriverOptions["fstype"]; ok { diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 58ff1a4e89..5226567df7 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -1180,7 +1180,6 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat } case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI: - rootfs.Driver = kataSCSIDevType rootfs.Source = blockDrive.SCSIAddr default: @@ -1195,8 +1194,8 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat } // Ensure container mount destination exists - // TODO: remove dependency on shared fs path. shared fs is just one kind of storage sources. - // we should not always use shared fs path for all kinds of storage. Stead, all storage + // TODO: remove dependency on shared fs path. shared fs is just one kind of storage source. + // we should not always use shared fs path for all kinds of storage. Instead, all storage // should be bind mounted to a tmpfs path for containers to use. if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil { return nil, err @@ -1204,13 +1203,10 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat return rootfs, nil } - // This is not a block based device rootfs. - // We are going to bind mount it into the 9pfs - // shared drive between the host and the guest. - // With 9pfs we don't need to ask the agent to - // mount the rootfs as the shared directory - // (kataGuestSharedDir) is already mounted in the - // guest. We only need to mount the rootfs from + // This is not a block based device rootfs. We are going to bind mount it into the shared drive + // between the host and the guest. + // With virtiofs/9pfs we don't need to ask the agent to mount the rootfs as the shared directory + // (kataGuestSharedDir) is already mounted in the guest. We only need to mount the rootfs from // the host and it will show up in the guest. if err := bindMountContainerRootfs(k.ctx, getMountPath(sandbox.id), c.id, c.rootFs.Target, false); err != nil { return nil, err @@ -1240,9 +1236,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, } }() + // setup rootfs -- if its block based, we'll receive a non-nil storage object representing + // the block device for the rootfs, which us utilized for mounting in the guest. This'll be handled + // already for non-block based rootfs if rootfs, err = k.buildContainerRootfs(sandbox, c, rootPathParent); err != nil { return nil, err - } else if rootfs != nil { + } + + if rootfs != nil { // Add rootfs to the list of container storage. // We only need to do this for block based rootfs, as we // want the agent to mount it into the right location @@ -1291,6 +1292,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process, if err != nil { return nil, err } + if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { return nil, err } @@ -1400,7 +1402,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r // handleDeviceBlockVolume handles volume that is block device file // and DeviceBlock type. -func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*grpc.Storage, error) { +func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) { vol := &grpc.Storage{} blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive) @@ -1435,12 +1437,22 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*g return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver) } + vol.MountPoint = m.Destination + + // If no explicit FS Type or Options are being set, then let's use what is provided for the particular mount: + if vol.Fstype == "" { + vol.Fstype = m.Type + } + if len(vol.Options) == 0 { + vol.Options = m.Options + } + return vol, nil } // handleVhostUserBlkVolume handles volume that is block device file // and VhostUserBlk type. -func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*grpc.Storage, error) { +func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) { vol := &grpc.Storage{} d, ok := device.GetDeviceInfo().(*config.VhostUserDeviceAttrs) @@ -1451,6 +1463,9 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (* vol.Driver = kataBlkDevType vol.Source = d.PCIAddr + vol.Fstype = "bind" + vol.Options = []string{"bind"} + vol.MountPoint = m.Destination return vol, nil } @@ -1483,9 +1498,9 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { var err error switch device.DeviceType() { case config.DeviceBlock: - vol, err = k.handleDeviceBlockVolume(c, device) + vol, err = k.handleDeviceBlockVolume(c, m, device) case config.VhostUserBlk: - vol, err = k.handleVhostUserBlkVolume(c, device) + vol, err = k.handleVhostUserBlkVolume(c, m, device) default: k.Logger().Error("Unknown device type") continue @@ -1495,14 +1510,6 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { return nil, err } - vol.MountPoint = m.Destination - if vol.Fstype == "" { - vol.Fstype = "bind" - } - if len(vol.Options) == 0 { - vol.Options = []string{"bind"} - } - volumeStorages = append(volumeStorages, vol) } diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 5a7bcf6d3f..7a7cb73fec 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -228,6 +228,7 @@ func TestHandleDeviceBlockVolume(t *testing.T) { tests := []struct { BlockDeviceDriver string + inputMount Mount inputDev *drivers.BlockDevice resultVol *pb.Storage }{ @@ -239,6 +240,7 @@ func TestHandleDeviceBlockVolume(t *testing.T) { Format: testBlkDriveFormat, }, }, + inputMount: Mount{}, resultVol: &pb.Storage{ Driver: kataNvdimmDevType, Source: fmt.Sprintf("/dev/pmem%s", testNvdimmID), @@ -248,18 +250,25 @@ func TestHandleDeviceBlockVolume(t *testing.T) { }, { BlockDeviceDriver: config.VirtioBlockCCW, + inputMount: Mount{ + Type: "bind", + Options: []string{"ro"}, + }, inputDev: &drivers.BlockDevice{ BlockDrive: &config.BlockDrive{ DevNo: testDevNo, }, }, resultVol: &pb.Storage{ - Driver: kataBlkCCWDevType, - Source: testDevNo, + Driver: kataBlkCCWDevType, + Source: testDevNo, + Fstype: "bind", + Options: []string{"ro"}, }, }, { BlockDeviceDriver: config.VirtioBlock, + inputMount: Mount{}, inputDev: &drivers.BlockDevice{ BlockDrive: &config.BlockDrive{ PCIAddr: testPCIAddr, @@ -320,7 +329,7 @@ func TestHandleDeviceBlockVolume(t *testing.T) { }, } - vol, _ := k.handleDeviceBlockVolume(c, test.inputDev) + vol, _ := k.handleDeviceBlockVolume(c, test.inputMount, test.inputDev) assert.True(t, reflect.DeepEqual(vol, test.resultVol), "Volume didn't match: got %+v, expecting %+v", vol, test.resultVol) @@ -336,22 +345,27 @@ func TestHandleBlockVolume(t *testing.T) { containers := map[string]*Container{} containers[c.id] = c - // Create a VhostUserBlk device and a DeviceBlock device + // Create a devices for VhostUserBlk, standard DeviceBlock and direct assigned Block device vDevID := "MockVhostUserBlk" bDevID := "MockDeviceBlock" + dDevID := "MockDeviceBlockDirect" vDestination := "/VhostUserBlk/destination" bDestination := "/DeviceBlock/destination" + dDestination := "/DeviceDirectBlock/destination" vPCIAddr := "0001:01" bPCIAddr := "0002:01" + dPCIAddr := "0003:01" vDev := drivers.NewVhostUserBlkDevice(&config.DeviceInfo{ID: vDevID}) bDev := drivers.NewBlockDevice(&config.DeviceInfo{ID: bDevID}) + dDev := drivers.NewBlockDevice(&config.DeviceInfo{ID: dDevID}) vDev.VhostUserDeviceAttrs = &config.VhostUserDeviceAttrs{PCIAddr: vPCIAddr} bDev.BlockDrive = &config.BlockDrive{PCIAddr: bPCIAddr} + dDev.BlockDrive = &config.BlockDrive{PCIAddr: dPCIAddr} var devices []api.Device - devices = append(devices, vDev, bDev) + devices = append(devices, vDev, bDev, dDev) // Create a VhostUserBlk mount and a DeviceBlock mount var mounts []Mount @@ -362,8 +376,16 @@ func TestHandleBlockVolume(t *testing.T) { bMount := Mount{ BlockDeviceID: bDevID, Destination: bDestination, + Type: "bind", + Options: []string{"bind"}, } - mounts = append(mounts, vMount, bMount) + dMount := Mount{ + BlockDeviceID: dDevID, + Destination: dDestination, + Type: "ext4", + Options: []string{"ro"}, + } + mounts = append(mounts, vMount, bMount, dMount) tmpDir := "/vhost/user/dir" dm := manager.NewDeviceManager(manager.VirtioBlock, true, tmpDir, devices) @@ -398,9 +420,17 @@ func TestHandleBlockVolume(t *testing.T) { Driver: kataBlkDevType, Source: bPCIAddr, } + dStorage := &pb.Storage{ + MountPoint: dDestination, + Fstype: "ext4", + Options: []string{"ro"}, + Driver: kataBlkDevType, + Source: dPCIAddr, + } assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume") assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume") + assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume") } func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { diff --git a/src/runtime/virtcontainers/qemu.go b/src/runtime/virtcontainers/qemu.go index 88e5ab71e7..c64e10f041 100644 --- a/src/runtime/virtcontainers/qemu.go +++ b/src/runtime/virtcontainers/qemu.go @@ -24,7 +24,7 @@ import ( "time" "unsafe" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/opencontainers/selinux/go-selinux/label" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -1096,9 +1096,9 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev } if q.config.BlockDeviceCacheSet { - err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush) + err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush, drive.ReadOnly) } else { - err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID) + err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID, drive.ReadOnly) } if err != nil { return err diff --git a/src/runtime/virtcontainers/qemu_amd64.go b/src/runtime/virtcontainers/qemu_amd64.go index b043a9e0c3..bc02dcc4d3 100644 --- a/src/runtime/virtcontainers/qemu_amd64.go +++ b/src/runtime/virtcontainers/qemu_amd64.go @@ -11,7 +11,7 @@ import ( "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" ) type qemuAmd64 struct { @@ -109,7 +109,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) { var q35QemuIOMMUOptions = "accel=kvm,kernel_irqchip=split" kernelParams = append(kernelParams, - Param{"intel_iommu", "on"}) + Param{"kata-containers_iommu", "on"}) kernelParams = append(kernelParams, Param{"iommu", "pt"}) diff --git a/src/runtime/virtcontainers/qemu_amd64_test.go b/src/runtime/virtcontainers/qemu_amd64_test.go index ba810299f7..c729e04cb5 100644 --- a/src/runtime/virtcontainers/qemu_amd64_test.go +++ b/src/runtime/virtcontainers/qemu_amd64_test.go @@ -11,7 +11,7 @@ import ( "os" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/stretchr/testify/assert" ) @@ -255,7 +255,7 @@ func TestQemuAmd64Iommu(t *testing.T) { assert.NoError(err) p := qemu.kernelParameters(false) - assert.Contains(p, Param{"intel_iommu", "on"}) + assert.Contains(p, Param{"kata-containers_iommu", "on"}) m := qemu.machine() assert.Contains(m.Options, "kernel_irqchip=split") diff --git a/src/runtime/virtcontainers/qemu_arch_base.go b/src/runtime/virtcontainers/qemu_arch_base.go index ba1f66b52a..7254631f3a 100644 --- a/src/runtime/virtcontainers/qemu_arch_base.go +++ b/src/runtime/virtcontainers/qemu_arch_base.go @@ -14,7 +14,7 @@ import ( "strconv" "strings" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" diff --git a/src/runtime/virtcontainers/qemu_arch_base_test.go b/src/runtime/virtcontainers/qemu_arch_base_test.go index d4a3b345a2..fcde7ec6b2 100644 --- a/src/runtime/virtcontainers/qemu_arch_base_test.go +++ b/src/runtime/virtcontainers/qemu_arch_base_test.go @@ -13,7 +13,7 @@ import ( "path/filepath" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/stretchr/testify/assert" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" diff --git a/src/runtime/virtcontainers/qemu_arm64.go b/src/runtime/virtcontainers/qemu_arm64.go index daef7fa3ff..14fdc4627e 100644 --- a/src/runtime/virtcontainers/qemu_arm64.go +++ b/src/runtime/virtcontainers/qemu_arm64.go @@ -10,7 +10,7 @@ import ( "fmt" "time" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) diff --git a/src/runtime/virtcontainers/qemu_arm64_test.go b/src/runtime/virtcontainers/qemu_arm64_test.go index 95bffacc26..8e5b78077a 100644 --- a/src/runtime/virtcontainers/qemu_arm64_test.go +++ b/src/runtime/virtcontainers/qemu_arm64_test.go @@ -11,7 +11,7 @@ import ( "os" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_ppc64le.go b/src/runtime/virtcontainers/qemu_ppc64le.go index 86fc141d79..0a8bbeb6b2 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le.go +++ b/src/runtime/virtcontainers/qemu_ppc64le.go @@ -9,7 +9,7 @@ import ( "fmt" "time" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" "github.com/sirupsen/logrus" ) diff --git a/src/runtime/virtcontainers/qemu_ppc64le_test.go b/src/runtime/virtcontainers/qemu_ppc64le_test.go index f48e2bae0b..c5e1038de5 100644 --- a/src/runtime/virtcontainers/qemu_ppc64le_test.go +++ b/src/runtime/virtcontainers/qemu_ppc64le_test.go @@ -9,7 +9,7 @@ import ( "fmt" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_s390x.go b/src/runtime/virtcontainers/qemu_s390x.go index a43fe4bff8..d829efa9d3 100644 --- a/src/runtime/virtcontainers/qemu_s390x.go +++ b/src/runtime/virtcontainers/qemu_s390x.go @@ -9,7 +9,7 @@ import ( "fmt" "time" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" ) diff --git a/src/runtime/virtcontainers/qemu_s390x_test.go b/src/runtime/virtcontainers/qemu_s390x_test.go index 4cdeeb245e..274329054b 100644 --- a/src/runtime/virtcontainers/qemu_s390x_test.go +++ b/src/runtime/virtcontainers/qemu_s390x_test.go @@ -9,7 +9,7 @@ import ( "fmt" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/stretchr/testify/assert" ) diff --git a/src/runtime/virtcontainers/qemu_test.go b/src/runtime/virtcontainers/qemu_test.go index 15e4de64c1..80ba64109c 100644 --- a/src/runtime/virtcontainers/qemu_test.go +++ b/src/runtime/virtcontainers/qemu_test.go @@ -14,7 +14,7 @@ import ( "strings" "testing" - govmmQemu "github.com/intel/govmm/qemu" + govmmQemu "github.com/kata-containers/govmm/qemu" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types" diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index 16db127ea1..cfa4cec8d1 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -1183,13 +1183,13 @@ func (s *Sandbox) fetchContainers() error { // This should be called only when the sandbox is already created. // It will add new container config to sandbox.config.Containers func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, error) { - // Create the container. + // Create the container object, add devices to the sandbox's device-manager: c, err := newContainer(s, &contConfig) if err != nil { return nil, err } - // Update sandbox config. + // Update sandbox config to include the new container's config s.config.Containers = append(s.config.Containers, contConfig) defer func() { @@ -1201,6 +1201,7 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro } }() + // create and start the container err = c.create() if err != nil { return nil, err