Merge pull request #1258 from egernst/ro-stable

volume cleanup, RO blk device support

src/runtime/go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/gogo/googleapis v1.4.0 // indirect
 	github.com/gogo/protobuf v1.3.1
 	github.com/hashicorp/go-multierror v1.0.0
-	github.com/intel/govmm v0.0.0-20200825065022-6042f6033126
+	github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca
 	github.com/mdlayher/vsock v0.0.0-20191108225356-d9c65923cb8f
 	github.com/opencontainers/image-spec v1.0.1 // indirect
 	github.com/opencontainers/runc v1.0.0-rc9.0.20200102164712-2b52db75279c

src/runtime/go.sum

@@ -164,12 +164,6 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/intel/govmm v0.0.0-20200724170648-af9e34b91ae9 h1:GSRnVLRNweZV3f8Vo3vtMlPsuzvpj57Gj7Y7TkGVO6U=
-github.com/intel/govmm v0.0.0-20200724170648-af9e34b91ae9/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g=
-github.com/intel/govmm v0.0.0-20200728135209-6c3315ba8a42 h1:Yu3/MlZl/kKE0Ipgio/KVorMIrjeHTVOG4+9WAddgOQ=
-github.com/intel/govmm v0.0.0-20200728135209-6c3315ba8a42/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g=
-github.com/intel/govmm v0.0.0-20200825065022-6042f6033126 h1:yltaUdR0Vitnn/FEfy+JWbJ+oGhMAPP/3S7ja9S5yso=
-github.com/intel/govmm v0.0.0-20200825065022-6042f6033126/go.mod h1:QKGWoQtjvkvFtzP6ybiM3lxUHqf83Sv3oLqyELUKH4g=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
@@ -179,7 +173,8 @@ github.com/juju/errors v0.0.0-20180806074554-22422dad46e1/go.mod h1:W54LbzXuIE0b
 github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/juju/testing v0.0.0-20190613124551-e81189438503/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kata-containers/kata-containers v0.0.0-20201013034856-c88820454d08 h1:yk9fzLKb9RmV9xuT5mkJw4owk/K0rX5cusm2ukEEDro=
+github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca h1:UdXFthwasAPnmv37gLJUEFsW9FaabYA+mM6FoSi8kzU=
+github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca/go.mod h1:VmAHbsL5lLfzHW/MNL96NVLF840DNEV5i683kISgFKk=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=

src/runtime/pkg/katautils/config.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 
 	"github.com/BurntSushi/toml"
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	vc "github.com/kata-containers/kata-containers/src/runtime/virtcontainers"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	exp "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/experimental"

src/runtime/vendor/github.com/intel/govmm/CONTRIBUTORS.md

@@ -1,22 +0,0 @@
-This file is a partial list of contributors to the Virtual Machine
-Manager for Go project. To see the full list of contributors, see the
-revision history in source control.
-
-Contributors who wish to be recognized in this file should add
-themselves (or their employer, as appropriate).
-
-- afrosi@de.ibm.com
-- archana.m.shinde@intel.com
-- caoruidong@huawei.com
-- clare.chenhui@huawei.com
-- eric.ernst@intel.com
-- james.o.hunt@intel.com
-- jose.carlos.venegas.munoz@intel.com
-- julio.montes@intel.com
-- manohar.r.castelino@intel.com
-- mark.d.ryan@intel.com
-- robert.bradford@intel.com
-- sameo@linux.intel.com
-- sebastien.boeuf@intel.com
-- teawater@hyper.sh
-- xinda.zhao@intel.com

src/runtime/vendor/github.com/kata-containers/govmm/qemu/qemu.go

@@ -135,7 +135,7 @@ const (
 
 func isDimmSupported(config *Config) bool {
 	switch runtime.GOARCH {
-	case "amd64", "386", "ppc64le":
+	case "amd64", "386", "ppc64le", "arm64":
 		if config != nil && config.Machine.Type == MachineTypeMicrovm {
 			// microvm does not support NUMA
 			return false
@@ -1084,6 +1084,8 @@ func (blkdev BlockDevice) QemuParams(config *Config) []string {
 		deviceParams = append(deviceParams, fmt.Sprintf(",share-rw=on"))
 	}
 
+	deviceParams = append(deviceParams, fmt.Sprintf(",serial=%s", blkdev.ID))
+
 	blkParams = append(blkParams, fmt.Sprintf("id=%s", blkdev.ID))
 	blkParams = append(blkParams, fmt.Sprintf(",file=%s", blkdev.File))
 	blkParams = append(blkParams, fmt.Sprintf(",aio=%s", blkdev.AIO))
@@ -1118,6 +1120,24 @@ func (blkdev BlockDevice) deviceName(config *Config) string {
 	return string(blkdev.Driver)
 }
 
+// PVPanicDevice represents a qemu pvpanic device.
+type PVPanicDevice struct {
+	NoShutdown bool
+}
+
+// Valid always returns true for pvpanic device
+func (dev PVPanicDevice) Valid() bool {
+	return true
+}
+
+// QemuParams returns the qemu parameters built out of this serial device.
+func (dev PVPanicDevice) QemuParams(config *Config) []string {
+	if dev.NoShutdown {
+		return []string{"-device", "pvpanic", "-no-shutdown"}
+	}
+	return []string{"-device", "pvpanic"}
+}
+
 // VhostUserDevice represents a qemu vhost-user device meant to be passed
 // in to the guest
 type VhostUserDevice struct {
@@ -2103,6 +2123,56 @@ type Kernel struct {
 	Params string
 }
 
+// FwCfg allows QEMU to pass entries to the guest
+// File and Str are mutually exclusive
+type FwCfg struct {
+	Name string
+	File string
+	Str  string
+}
+
+// Valid returns true if the FwCfg structure is valid and complete.
+func (fwcfg FwCfg) Valid() bool {
+	if fwcfg.Name == "" {
+		return false
+	}
+
+	if fwcfg.File != "" && fwcfg.Str != "" {
+		return false
+	}
+
+	if fwcfg.File == "" && fwcfg.Str == "" {
+		return false
+	}
+
+	return true
+}
+
+// QemuParams returns the qemu parameters built out of the FwCfg object
+func (fwcfg FwCfg) QemuParams(config *Config) []string {
+	var fwcfgParams []string
+	var qemuParams []string
+
+	for _, f := range config.FwCfg {
+		if f.Name != "" {
+			fwcfgParams = append(fwcfgParams, fmt.Sprintf("name=%s", f.Name))
+
+			if f.File != "" {
+				fwcfgParams = append(fwcfgParams, fmt.Sprintf(",file=%s", f.File))
+			}
+
+			if f.Str != "" {
+				fwcfgParams = append(fwcfgParams, fmt.Sprintf(",string=%s", f.Str))
+			}
+		}
+
+		qemuParams = append(qemuParams, "-fw_cfg")
+		qemuParams = append(qemuParams, strings.Join(fwcfgParams, ""))
+	}
+
+	return qemuParams
+}
+
 // Knobs regroups a set of qemu boolean settings
 type Knobs struct {
 	// NoUserConfig prevents qemu from loading user config files.
@@ -2230,12 +2300,18 @@ type Config struct {
 	// Bios is the -bios parameter
 	Bios string
 
+	// PFlash specifies the parallel flash images (-pflash parameter)
+	PFlash []string
+
 	// Incoming controls migration source preparation
 	Incoming Incoming
 
 	// fds is a list of open file descriptors to be passed to the spawned qemu process
 	fds []*os.File
 
+	// FwCfg is the -fw_cfg parameter
+	FwCfg []FwCfg
+
 	IOThreads []IOThread
 
 	// PidFile is the -pidfile parameter
@@ -2417,6 +2493,13 @@ func (config *Config) appendGlobalParam() {
 	}
 }
 
+func (config *Config) appendPFlashParam() {
+	for _, p := range config.PFlash {
+		config.qemuParams = append(config.qemuParams, "-pflash")
+		config.qemuParams = append(config.qemuParams, p)
+	}
+}
+
 func (config *Config) appendVGA() {
 	if config.VGA != "" {
 		config.qemuParams = append(config.qemuParams, "-vga")
@@ -2568,6 +2651,21 @@ func (config *Config) appendLogFile() {
 	}
 }
 
+func (config *Config) appendFwCfg(logger QMPLog) {
+	if logger == nil {
+		logger = qmpNullLogger{}
+	}
+
+	for _, f := range config.FwCfg {
+		if !f.Valid() {
+			logger.Errorf("fw_cfg is not valid: %+v", config.FwCfg)
+			continue
+		}
+
+		config.qemuParams = append(config.qemuParams, f.QemuParams(config)...)
+	}
+}
+
 // LaunchQemu can be used to launch a new qemu instance.
 //
 // The Config parameter contains a set of qemu parameters and settings.
@@ -2587,6 +2685,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) {
 	config.appendDevices()
 	config.appendRTC()
 	config.appendGlobalParam()
+	config.appendPFlashParam()
 	config.appendVGA()
 	config.appendKnobs()
 	config.appendKernel()
@@ -2595,6 +2694,7 @@ func LaunchQemu(config Config, logger QMPLog) (string, error) {
 	config.appendIncoming()
 	config.appendPidFile()
 	config.appendLogFile()
+	config.appendFwCfg(logger)
 
 	if err := config.appendCPUs(); err != nil {
 		return "", err

src/runtime/vendor/github.com/kata-containers/govmm/qemu/qmp.go

@@ -268,7 +268,7 @@ func (q *QMP) readLoop(fromVMCh chan<- []byte) {
 	for scanner.Scan() {
 		line := scanner.Bytes()
 		if q.cfg.Logger.V(1) {
-			q.cfg.Logger.Infof("%s", string(line))
+			q.cfg.Logger.Infof("read from QMP: %s", string(line))
 		}
 
 		// Since []byte channel type transfer slice info(include slice underlying array pointer, len, cap)
@@ -773,11 +773,12 @@ func (q *QMP) ExecuteQuit(ctx context.Context) error {
 	return q.executeCommand(ctx, "quit", nil, nil)
 }
 
-func (q *QMP) blockdevAddBaseArgs(device, blockdevID string) (map[string]interface{}, map[string]interface{}) {
+func (q *QMP) blockdevAddBaseArgs(device, blockdevID string, ro bool) (map[string]interface{}, map[string]interface{}) {
 	var args map[string]interface{}
 
 	blockdevArgs := map[string]interface{}{
-		"driver": "raw",
+		"driver":    "raw",
+		"read-only": ro,
 		"file": map[string]interface{}{
 			"driver":   "file",
 			"filename": device,
@@ -801,8 +802,8 @@ func (q *QMP) blockdevAddBaseArgs(device, blockdevID string) (map[string]interfa
 // path of the device to add, e.g., /dev/rdb0, and blockdevID is an identifier
 // used to name the device.  As this identifier will be passed directly to QMP,
 // it must obey QMP's naming rules, e,g., it must start with a letter.
-func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string) error {
-	args, _ := q.blockdevAddBaseArgs(device, blockdevID)
+func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string, ro bool) error {
+	args, _ := q.blockdevAddBaseArgs(device, blockdevID, ro)
 
 	return q.executeCommand(ctx, "blockdev-add", args, nil)
 }
@@ -814,8 +815,8 @@ func (q *QMP) ExecuteBlockdevAdd(ctx context.Context, device, blockdevID string)
 // direct denotes whether use of O_DIRECT (bypass the host page cache)
 // is enabled.  noFlush denotes whether flush requests for the device are
 // ignored.
-func (q *QMP) ExecuteBlockdevAddWithCache(ctx context.Context, device, blockdevID string, direct, noFlush bool) error {
-	args, blockdevArgs := q.blockdevAddBaseArgs(device, blockdevID)
+func (q *QMP) ExecuteBlockdevAddWithCache(ctx context.Context, device, blockdevID string, direct, noFlush, ro bool) error {
+	args, blockdevArgs := q.blockdevAddBaseArgs(device, blockdevID, ro)
 
 	if q.version.Major < 2 || (q.version.Major == 2 && q.version.Minor < 9) {
 		return fmt.Errorf("versions of qemu (%d.%d) older than 2.9 do not support set cache-related options for block devices",
@@ -1639,3 +1640,29 @@ func (q *QMP) ExecQomSet(ctx context.Context, path, property string, value uint6
 
 	return q.executeCommand(ctx, "qom-set", args, nil)
 }
+
+// ExecQomGet qom-get path property
+func (q *QMP) ExecQomGet(ctx context.Context, path, property string) (interface{}, error) {
+	args := map[string]interface{}{
+		"path":     path,
+		"property": property,
+	}
+
+	response, err := q.executeCommandWithResponse(ctx, "qom-get", args, nil, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return response, nil
+}
+
+// ExecuteDumpGuestMemory dump guest memory to host
+func (q *QMP) ExecuteDumpGuestMemory(ctx context.Context, protocol string, paging bool, format string) error {
+	args := map[string]interface{}{
+		"protocol": protocol,
+		"paging":   paging,
+		"format":   format,
+	}
+
+	return q.executeCommand(ctx, "dump-guest-memory", args, nil)
+}

src/runtime/vendor/modules.txt

@@ -222,9 +222,9 @@ github.com/hashicorp/errwrap
 # github.com/hashicorp/go-multierror v1.0.0
 ## explicit
 github.com/hashicorp/go-multierror
-# github.com/intel/govmm v0.0.0-20200825065022-6042f6033126
+# github.com/kata-containers/govmm v0.0.0-20210112013750-7d320e8f5dca
 ## explicit
-github.com/intel/govmm/qemu
+github.com/kata-containers/govmm/qemu
 # github.com/konsorten/go-windows-terminal-sequences v1.0.1
 github.com/konsorten/go-windows-terminal-sequences
 # github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329

src/runtime/virtcontainers/container.go

@@ -481,7 +481,7 @@ func (c *Container) shareFiles(m Mount, idx int, hostSharedDir, guestSharedDir s
 }
 
 // mountSharedDirMounts handles bind-mounts by bindmounting to the host shared
-// directory which is mounted through 9pfs in the VM.
+// directory which is mounted through virtiofs/9pfs in the VM.
 // It also updates the container mount list with the HostPath info, and store
 // container mounts to the storage. This way, we will have the HostPath info
 // available when we will need to unmount those mounts.
@@ -504,6 +504,18 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 			continue
 		}
 
+		// Check if mount is a block device file. If it is, the block device will be attached to the host
+		// instead of passing this as a shared mount:
+		if len(m.BlockDeviceID) > 0 {
+			// Attach this block device, all other devices passed in the config have been attached at this point
+			if err = c.sandbox.devManager.AttachDevice(m.BlockDeviceID, c.sandbox); err != nil {
+				return nil, nil, err
+			}
+			devicesToDetach = append(devicesToDetach, m.BlockDeviceID)
+			continue
+		}
+
+		// For non-block based mounts, we are only interested in bind mounts
 		if m.Type != "bind" {
 			continue
 		}
@@ -515,17 +527,6 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) (
 			continue
 		}
 
-		// Check if mount is a block device file. If it is, the block device will be attached to the host
-		// instead of passing this as a shared mount.
-		if len(m.BlockDeviceID) > 0 {
-			// Attach this block device, all other devices passed in the config have been attached at this point
-			if err = c.sandbox.devManager.AttachDevice(m.BlockDeviceID, c.sandbox); err != nil {
-				return nil, nil, err
-			}
-			devicesToDetach = append(devicesToDetach, m.BlockDeviceID)
-			continue
-		}
-
 		// Ignore /dev, directories and all other device files. We handle
 		// only regular files in /dev. It does not make sense to pass the host
 		// device nodes to the guest.
@@ -631,6 +632,9 @@ func filterDevices(c *Container, devices []ContainerDevice) (ret []ContainerDevi
 	return
 }
 
+// Add any mount based block devices to the device manager and save the
+// device ID for the particular mount. This'll occur when the mountpoint source
+// is a block device.
 func (c *Container) createBlockDevices() error {
 	if !c.checkBlockDeviceSupport() {
 		c.Logger().Warn("Block device not supported")
@@ -639,13 +643,18 @@ func (c *Container) createBlockDevices() error {
 
 	// iterate all mounts and create block device if it's block based.
 	for i, m := range c.mounts {
-		if len(m.BlockDeviceID) > 0 || m.Type != "bind" {
+		if len(m.BlockDeviceID) > 0 {
 			// Non-empty m.BlockDeviceID indicates there's already one device
 			// associated with the mount,so no need to create a new device for it
 			// and we only create block device for bind mount
 			continue
 		}
 
+		if m.Type != "bind" {
+			// We only handle for bind-mounts
+			continue
+		}
+
 		var stat unix.Stat_t
 		if err := unix.Stat(m.Source, &stat); err != nil {
 			return fmt.Errorf("stat %q failed: %v", m.Source, err)
@@ -663,6 +672,7 @@ func (c *Container) createBlockDevices() error {
 				DevType:       "b",
 				Major:         int64(unix.Major(stat.Rdev)),
 				Minor:         int64(unix.Minor(stat.Rdev)),
+				ReadOnly:      m.ReadOnly,
 			}
 			// check whether source can be used as a pmem device
 		} else if di, err = config.PmemDeviceInfo(m.Source, m.Destination); err != nil {
@@ -673,7 +683,6 @@ func (c *Container) createBlockDevices() error {
 
 		if err == nil && di != nil {
 			b, err := c.sandbox.devManager.NewDevice(*di)
-
 			if err != nil {
 				// Do not return an error, try to create
 				// devices for other mounts
@@ -725,11 +734,12 @@ func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, er
 		return nil, err
 	}
 
-	// Go to next step for first created container
+	// If mounts are block devices, add to devmanager
 	if err := c.createMounts(); err != nil {
 		return nil, err
 	}
 
+	// Add container's devices to sandbox's device-manager
 	if err := c.createDevices(contConfig); err != nil {
 		return nil, err
 	}
@@ -739,11 +749,7 @@ func newContainer(sandbox *Sandbox, contConfig *ContainerConfig) (*Container, er
 
 func (c *Container) createMounts() error {
 	// Create block devices for newly created container
-	if err := c.createBlockDevices(); err != nil {
-		return err
-	}
-
-	return nil
+	return c.createBlockDevices()
 }
 
 func (c *Container) createDevices(contConfig *ContainerConfig) error {
@@ -813,6 +819,7 @@ func (c *Container) create() (err error) {
 	}()
 
 	if c.checkBlockDeviceSupport() {
+		// If the rootfs is backed by a block device, go ahead and hotplug it to the guest
 		if err = c.hotplugDrive(); err != nil {
 			return
 		}
@@ -1208,11 +1215,14 @@ func (c *Container) resume() error {
 	return c.setContainerState(types.StateRunning)
 }
 
+// hotplugDrive will attempt to hotplug the container rootfs if it is backed by a
+// block device
 func (c *Container) hotplugDrive() error {
 	var dev device
 	var err error
 
-	// container rootfs is blockdevice backed and isn't mounted
+	// Check to see if the rootfs is an umounted block device (source) or if the
+	// mount (target) is backed by a block device:
 	if !c.rootFs.Mounted {
 		dev, err = getDeviceForPath(c.rootFs.Source)
 		// there is no "rootfs" dir on block device backed rootfs
@@ -1274,6 +1284,7 @@ func (c *Container) hotplugDrive() error {
 	return c.setStateFstype(fsType)
 }
 
+// plugDevice will attach the rootfs if blockdevice is supported (this is rootfs specific)
 func (c *Container) plugDevice(devicePath string) error {
 	var stat unix.Stat_t
 	if err := unix.Stat(devicePath, &stat); err != nil {

src/runtime/virtcontainers/device/config/config.go

@@ -114,6 +114,9 @@ type DeviceInfo struct {
 	// for a nvdimm device in the guest.
 	Pmem bool
 
+	// If applicable, should this device be considered RO
+	ReadOnly bool
+
 	// ColdPlug specifies whether the device must be cold plugged (true)
 	// or hot plugged (false).
 	ColdPlug bool

src/runtime/virtcontainers/device/drivers/block.go

@@ -61,11 +61,12 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 	}
 
 	drive := &config.BlockDrive{
-		File:   device.DeviceInfo.HostPath,
-		Format: "raw",
-		ID:     utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize),
-		Index:  index,
-		Pmem:   device.DeviceInfo.Pmem,
+		File:     device.DeviceInfo.HostPath,
+		Format:   "raw",
+		ID:       utils.MakeNameID("drive", device.DeviceInfo.ID, maxDevIDSize),
+		Index:    index,
+		Pmem:     device.DeviceInfo.Pmem,
+		ReadOnly: device.DeviceInfo.ReadOnly,
 	}
 
 	if fs, ok := device.DeviceInfo.DriverOptions["fstype"]; ok {

src/runtime/virtcontainers/kata_agent.go

@@ -1180,7 +1180,6 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 			}
 
 		case sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioSCSI:
-
 			rootfs.Driver = kataSCSIDevType
 			rootfs.Source = blockDrive.SCSIAddr
 		default:
@@ -1195,8 +1194,8 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 		}
 
 		// Ensure container mount destination exists
-		// TODO: remove dependency on shared fs path. shared fs is just one kind of storage sources.
-		// we should not always use shared fs path for all kinds of storage. Stead, all storage
+		// TODO: remove dependency on shared fs path. shared fs is just one kind of storage source.
+		// we should not always use shared fs path for all kinds of storage. Instead, all storage
 		// should be bind mounted to a tmpfs path for containers to use.
 		if err := os.MkdirAll(filepath.Join(getMountPath(c.sandbox.id), c.id, c.rootfsSuffix), DirMode); err != nil {
 			return nil, err
@@ -1204,13 +1203,10 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 		return rootfs, nil
 	}
 
-	// This is not a block based device rootfs.
-	// We are going to bind mount it into the 9pfs
-	// shared drive between the host and the guest.
-	// With 9pfs we don't need to ask the agent to
-	// mount the rootfs as the shared directory
-	// (kataGuestSharedDir) is already mounted in the
-	// guest. We only need to mount the rootfs from
+	// This is not a block based device rootfs. We are going to bind mount it into the shared drive
+	// between the host and the guest.
+	// With virtiofs/9pfs we don't need to ask the agent to mount the rootfs as the shared directory
+	// (kataGuestSharedDir) is already mounted in the guest. We only need to mount the rootfs from
 	// the host and it will show up in the guest.
 	if err := bindMountContainerRootfs(k.ctx, getMountPath(sandbox.id), c.id, c.rootFs.Target, false); err != nil {
 		return nil, err
@@ -1240,9 +1236,14 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 		}
 	}()
 
+	// setup rootfs -- if its block based, we'll receive a non-nil storage object representing
+	// the block device for the rootfs, which us utilized for mounting in the guest. This'll be handled
+	// already for non-block based rootfs
 	if rootfs, err = k.buildContainerRootfs(sandbox, c, rootPathParent); err != nil {
 		return nil, err
-	} else if rootfs != nil {
+	}
+
+	if rootfs != nil {
 		// Add rootfs to the list of container storage.
 		// We only need to do this for block based rootfs, as we
 		// want the agent to mount it into the right location
@@ -1291,6 +1292,7 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 	if err != nil {
 		return nil, err
 	}
+
 	if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
 		return nil, err
 	}
@@ -1400,7 +1402,7 @@ func (k *kataAgent) handleLocalStorage(mounts []specs.Mount, sandboxID string, r
 
 // handleDeviceBlockVolume handles volume that is block device file
 // and DeviceBlock type.
-func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*grpc.Storage, error) {
+func (k *kataAgent) handleDeviceBlockVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) {
 	vol := &grpc.Storage{}
 
 	blockDrive, ok := device.GetDeviceInfo().(*config.BlockDrive)
@@ -1435,12 +1437,22 @@ func (k *kataAgent) handleDeviceBlockVolume(c *Container, device api.Device) (*g
 		return nil, fmt.Errorf("Unknown block device driver: %s", c.sandbox.config.HypervisorConfig.BlockDeviceDriver)
 	}
 
+	vol.MountPoint = m.Destination
+
+	// If no explicit FS Type or Options are being set, then let's use what is provided for the particular mount:
+	if vol.Fstype == "" {
+		vol.Fstype = m.Type
+	}
+	if len(vol.Options) == 0 {
+		vol.Options = m.Options
+	}
+
 	return vol, nil
 }
 
 // handleVhostUserBlkVolume handles volume that is block device file
 // and VhostUserBlk type.
-func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*grpc.Storage, error) {
+func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.Device) (*grpc.Storage, error) {
 	vol := &grpc.Storage{}
 
 	d, ok := device.GetDeviceInfo().(*config.VhostUserDeviceAttrs)
@@ -1451,6 +1463,9 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, device api.Device) (*
 
 	vol.Driver = kataBlkDevType
 	vol.Source = d.PCIAddr
+	vol.Fstype = "bind"
+	vol.Options = []string{"bind"}
+	vol.MountPoint = m.Destination
 
 	return vol, nil
 }
@@ -1483,9 +1498,9 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
 		var err error
 		switch device.DeviceType() {
 		case config.DeviceBlock:
-			vol, err = k.handleDeviceBlockVolume(c, device)
+			vol, err = k.handleDeviceBlockVolume(c, m, device)
 		case config.VhostUserBlk:
-			vol, err = k.handleVhostUserBlkVolume(c, device)
+			vol, err = k.handleVhostUserBlkVolume(c, m, device)
 		default:
 			k.Logger().Error("Unknown device type")
 			continue
@@ -1495,14 +1510,6 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
 			return nil, err
 		}
 
-		vol.MountPoint = m.Destination
-		if vol.Fstype == "" {
-			vol.Fstype = "bind"
-		}
-		if len(vol.Options) == 0 {
-			vol.Options = []string{"bind"}
-		}
-
 		volumeStorages = append(volumeStorages, vol)
 	}
 

src/runtime/virtcontainers/kata_agent_test.go

@@ -228,6 +228,7 @@ func TestHandleDeviceBlockVolume(t *testing.T) {
 
 	tests := []struct {
 		BlockDeviceDriver string
+		inputMount        Mount
 		inputDev          *drivers.BlockDevice
 		resultVol         *pb.Storage
 	}{
@@ -239,6 +240,7 @@ func TestHandleDeviceBlockVolume(t *testing.T) {
 					Format:   testBlkDriveFormat,
 				},
 			},
+			inputMount: Mount{},
 			resultVol: &pb.Storage{
 				Driver:  kataNvdimmDevType,
 				Source:  fmt.Sprintf("/dev/pmem%s", testNvdimmID),
@@ -248,18 +250,25 @@ func TestHandleDeviceBlockVolume(t *testing.T) {
 		},
 		{
 			BlockDeviceDriver: config.VirtioBlockCCW,
+			inputMount: Mount{
+				Type:    "bind",
+				Options: []string{"ro"},
+			},
 			inputDev: &drivers.BlockDevice{
 				BlockDrive: &config.BlockDrive{
 					DevNo: testDevNo,
 				},
 			},
 			resultVol: &pb.Storage{
-				Driver: kataBlkCCWDevType,
-				Source: testDevNo,
+				Driver:  kataBlkCCWDevType,
+				Source:  testDevNo,
+				Fstype:  "bind",
+				Options: []string{"ro"},
 			},
 		},
 		{
 			BlockDeviceDriver: config.VirtioBlock,
+			inputMount:        Mount{},
 			inputDev: &drivers.BlockDevice{
 				BlockDrive: &config.BlockDrive{
 					PCIAddr:  testPCIAddr,
@@ -320,7 +329,7 @@ func TestHandleDeviceBlockVolume(t *testing.T) {
 			},
 		}
 
-		vol, _ := k.handleDeviceBlockVolume(c, test.inputDev)
+		vol, _ := k.handleDeviceBlockVolume(c, test.inputMount, test.inputDev)
 		assert.True(t, reflect.DeepEqual(vol, test.resultVol),
 			"Volume didn't match: got %+v, expecting %+v",
 			vol, test.resultVol)
@@ -336,22 +345,27 @@ func TestHandleBlockVolume(t *testing.T) {
 	containers := map[string]*Container{}
 	containers[c.id] = c
 
-	// Create a VhostUserBlk device and a DeviceBlock device
+	// Create a devices for VhostUserBlk, standard DeviceBlock and direct assigned Block device
 	vDevID := "MockVhostUserBlk"
 	bDevID := "MockDeviceBlock"
+	dDevID := "MockDeviceBlockDirect"
 	vDestination := "/VhostUserBlk/destination"
 	bDestination := "/DeviceBlock/destination"
+	dDestination := "/DeviceDirectBlock/destination"
 	vPCIAddr := "0001:01"
 	bPCIAddr := "0002:01"
+	dPCIAddr := "0003:01"
 
 	vDev := drivers.NewVhostUserBlkDevice(&config.DeviceInfo{ID: vDevID})
 	bDev := drivers.NewBlockDevice(&config.DeviceInfo{ID: bDevID})
+	dDev := drivers.NewBlockDevice(&config.DeviceInfo{ID: dDevID})
 
 	vDev.VhostUserDeviceAttrs = &config.VhostUserDeviceAttrs{PCIAddr: vPCIAddr}
 	bDev.BlockDrive = &config.BlockDrive{PCIAddr: bPCIAddr}
+	dDev.BlockDrive = &config.BlockDrive{PCIAddr: dPCIAddr}
 
 	var devices []api.Device
-	devices = append(devices, vDev, bDev)
+	devices = append(devices, vDev, bDev, dDev)
 
 	// Create a VhostUserBlk mount and a DeviceBlock mount
 	var mounts []Mount
@@ -362,8 +376,16 @@ func TestHandleBlockVolume(t *testing.T) {
 	bMount := Mount{
 		BlockDeviceID: bDevID,
 		Destination:   bDestination,
+		Type:          "bind",
+		Options:       []string{"bind"},
 	}
-	mounts = append(mounts, vMount, bMount)
+	dMount := Mount{
+		BlockDeviceID: dDevID,
+		Destination:   dDestination,
+		Type:          "ext4",
+		Options:       []string{"ro"},
+	}
+	mounts = append(mounts, vMount, bMount, dMount)
 
 	tmpDir := "/vhost/user/dir"
 	dm := manager.NewDeviceManager(manager.VirtioBlock, true, tmpDir, devices)
@@ -398,9 +420,17 @@ func TestHandleBlockVolume(t *testing.T) {
 		Driver:     kataBlkDevType,
 		Source:     bPCIAddr,
 	}
+	dStorage := &pb.Storage{
+		MountPoint: dDestination,
+		Fstype:     "ext4",
+		Options:    []string{"ro"},
+		Driver:     kataBlkDevType,
+		Source:     dPCIAddr,
+	}
 
 	assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume")
 	assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume")
+	assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume")
 }
 
 func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) {

src/runtime/virtcontainers/qemu.go

@@ -24,7 +24,7 @@ import (
 	"time"
 	"unsafe"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
@@ -1096,9 +1096,9 @@ func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, dev
 	}
 
 	if q.config.BlockDeviceCacheSet {
-		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush)
+		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush, drive.ReadOnly)
 	} else {
-		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID)
+		err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID, drive.ReadOnly)
 	}
 	if err != nil {
 		return err

src/runtime/virtcontainers/qemu_amd64.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 )
 
 type qemuAmd64 struct {
@@ -109,7 +109,7 @@ func newQemuArch(config HypervisorConfig) (qemuArch, error) {
 		var q35QemuIOMMUOptions = "accel=kvm,kernel_irqchip=split"
 
 		kernelParams = append(kernelParams,
-			Param{"intel_iommu", "on"})
+			Param{"kata-containers_iommu", "on"})
 		kernelParams = append(kernelParams,
 			Param{"iommu", "pt"})
 

src/runtime/virtcontainers/qemu_amd64_test.go

@@ -11,7 +11,7 @@ import (
 	"os"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/stretchr/testify/assert"
 )
@@ -255,7 +255,7 @@ func TestQemuAmd64Iommu(t *testing.T) {
 	assert.NoError(err)
 
 	p := qemu.kernelParameters(false)
-	assert.Contains(p, Param{"intel_iommu", "on"})
+	assert.Contains(p, Param{"kata-containers_iommu", "on"})
 
 	m := qemu.machine()
 	assert.Contains(m.Options, "kernel_irqchip=split")

src/runtime/virtcontainers/qemu_arch_base.go

@@ -14,7 +14,7 @@ import (
 	"strconv"
 	"strings"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"

src/runtime/virtcontainers/qemu_arch_base_test.go

@@ -13,7 +13,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"

src/runtime/virtcontainers/qemu_arm64.go

@@ -10,7 +10,7 @@ import (
 	"fmt"
 	"time"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )
 

src/runtime/virtcontainers/qemu_arm64_test.go

@@ -11,7 +11,7 @@ import (
 	"os"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/virtcontainers/qemu_ppc64le.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"time"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 	"github.com/sirupsen/logrus"
 )

src/runtime/virtcontainers/qemu_ppc64le_test.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/stretchr/testify/assert"
 )
 

src/runtime/virtcontainers/qemu_s390x.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"time"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"
 )

src/runtime/virtcontainers/qemu_s390x_test.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/stretchr/testify/assert"
 )

src/runtime/virtcontainers/qemu_test.go

@@ -14,7 +14,7 @@ import (
 	"strings"
 	"testing"
 
-	govmmQemu "github.com/intel/govmm/qemu"
+	govmmQemu "github.com/kata-containers/govmm/qemu"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/persist"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/types"

src/runtime/virtcontainers/sandbox.go

@@ -1183,13 +1183,13 @@ func (s *Sandbox) fetchContainers() error {
 // This should be called only when the sandbox is already created.
 // It will add new container config to sandbox.config.Containers
 func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, error) {
-	// Create the container.
+	// Create the container object, add devices to the sandbox's device-manager:
 	c, err := newContainer(s, &contConfig)
 	if err != nil {
 		return nil, err
 	}
 
-	// Update sandbox config.
+	// Update sandbox config to include the new container's config
 	s.config.Containers = append(s.config.Containers, contConfig)
 
 	defer func() {
@@ -1201,6 +1201,7 @@ func (s *Sandbox) CreateContainer(contConfig ContainerConfig) (VCContainer, erro
 		}
 	}()
 
+	// create and start the container
 	err = c.create()
 	if err != nil {
 		return nil, err