virtio-mmio: Add support for virtio-mmio

Start adding support for virtio-mmio devices starting with block.
The devices show within the vm as vda, vdb,... based on order of
insertion and such within the VM resemble virtio-blk devices.

They need to be explicitly differentiated to ensure that the
agent logic within the VM can discover and mount them appropropriately.
The agent uses PCI location to discover them for virtio-blk.
For virtio-mmio we need to use the predicted device name for now.

Note: Kata used a disk for the VM rootfs in the case of Firecracker.
(Instead of initrd or virtual-nvdimm). The Kata code today does not
handle this case properly.

For now as Firecracker is the only Hypervisor in Kata that
uses virtio-mmio directly offset the drive index to comprehend
this.

Longer term we should track if the rootfs is setup as a block
device explicitly.

Fixes: #1046

Signed-off-by: Sebastien Boeuf <sebastien.boeuf@intel.com>
Signed-off-by: Manohar Castelino <manohar.r.castelino@intel.com>

pkg/katautils/config.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/BurntSushi/toml"
 	vc "github.com/kata-containers/runtime/virtcontainers"
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
 	"github.com/kata-containers/runtime/virtcontainers/utils"
 	"github.com/sirupsen/logrus"
@@ -292,15 +293,19 @@ func (h hypervisor) defaultBridges() uint32 {
 }
 
 func (h hypervisor) blockDeviceDriver() (string, error) {
+	supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio}
+
 	if h.BlockDeviceDriver == "" {
 		return defaultBlockDeviceDriver, nil
 	}
 
-	if h.BlockDeviceDriver != vc.VirtioSCSI && h.BlockDeviceDriver != vc.VirtioBlock {
+	for _, b := range supportedBlockDrivers {
-		return "", fmt.Errorf("Invalid value %s provided for hypervisor block storage driver, can be either %s or %s", h.BlockDeviceDriver, vc.VirtioSCSI, vc.VirtioBlock)
+		if b == h.BlockDeviceDriver {
+			return h.BlockDeviceDriver, nil
+		}
 	}
 
-	return h.BlockDeviceDriver, nil
+	return "", fmt.Errorf("Invalid hypervisor block storage driver %v specified (supported drivers: %v)", h.BlockDeviceDriver, supportedBlockDrivers)
 }
 
 func (h hypervisor) msize9p() uint32 {

virtcontainers/device/config/config.go

@@ -38,6 +38,17 @@ const (
 	VhostUserBlk = "vhost-user-blk-pci"
 )
 
+const (
+	// VirtioMmio means use virtio-mmio for mmio based drives
+	VirtioMmio = "virtio-mmio"
+
+	// VirtioBlock means use virtio-blk for hotplugging drives
+	VirtioBlock = "virtio-blk"
+
+	// VirtioSCSI means use virtio-scsi for hotplugging drives
+	VirtioSCSI = "virtio-scsi"
+)
+
 // Defining these as a variable instead of a const, to allow
 // overriding this in the tests.
 
@@ -98,6 +109,9 @@ type BlockDrive struct {
 	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
 	Index int
 
+	// MmioAddr is used to identify the slot at which the drive is attached (order?).
+	MmioAddr string
+
 	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
 	PCIAddr string
 

virtcontainers/device/drivers/block.go

@@ -67,21 +67,39 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
 		Index:  index,
 	}
 
-	driveName, err := utils.GetVirtDriveName(index)
-	if err != nil {
-		return err
-	}
-
 	customOptions := device.DeviceInfo.DriverOptions
-	if customOptions != nil && customOptions["block-driver"] == "virtio-blk" {
+	if customOptions == nil ||
-		drive.VirtPath = filepath.Join("/dev", driveName)
+		customOptions["block-driver"] == "virtio-scsi" {
-	} else {
+		// User has not chosen a specific block device type
+		// Default to SCSI
 		scsiAddr, err := utils.GetSCSIAddress(index)
 		if err != nil {
 			return err
 		}
 
 		drive.SCSIAddr = scsiAddr
+	} else {
+		var globalIdx int
+
+		switch customOptions["block-driver"] {
+		case "virtio-blk":
+			globalIdx = index
+		case "virtio-mmio":
+			//With firecracker the rootfs for the VM itself
+			//sits at /dev/vda and consumes the first index.
+			//Longer term block based VM rootfs should be added
+			//as a regular block device which eliminates the
+			//offset.
+			//https://github.com/kata-containers/runtime/issues/1061
+			globalIdx = index + 1
+		}
+
+		driveName, err := utils.GetVirtDriveName(globalIdx)
+		if err != nil {
+			return err
+		}
+
+		drive.VirtPath = filepath.Join("/dev", driveName)
 	}
 
 	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")

virtcontainers/device/manager/manager.go

@@ -20,6 +20,8 @@ import (
 )
 
 const (
+	// VirtioMmio indicates block driver is virtio-mmio based
+	VirtioMmio string = "virtio-mmio"
 	// VirtioBlock indicates block driver is virtio-blk based
 	VirtioBlock string = "virtio-blk"
 	// VirtioSCSI indicates block driver is virtio-scsi based
@@ -55,7 +57,9 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
 	dm := &deviceManager{
 		devices: make(map[string]api.Device),
 	}
-	if blockDriver == VirtioBlock {
+	if blockDriver == VirtioMmio {
+		dm.blockDriver = VirtioMmio
+	} else if blockDriver == VirtioBlock {
 		dm.blockDriver = VirtioBlock
 	} else {
 		dm.blockDriver = VirtioSCSI

virtcontainers/hyperstart_agent.go

@@ -522,7 +522,8 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error {
 
 	if c.state.Fstype != "" {
 		// Pass a drive name only in case of block driver
-		if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
+		if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock ||
+			sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
 			driveName, err := utils.GetVirtDriveName(c.state.BlockIndex)
 			if err != nil {
 				return err

virtcontainers/hypervisor.go

@@ -13,6 +13,8 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+
+	"github.com/kata-containers/runtime/virtcontainers/device/config"
 )
 
 // HypervisorType describes an hypervisor type.
@@ -38,7 +40,7 @@ const (
 
 	defaultBridges = 1
 
-	defaultBlockDriver = VirtioSCSI
+	defaultBlockDriver = config.VirtioSCSI
 )
 
 // In some architectures the maximum number of vCPUs depends on the number of physical cores.

virtcontainers/kata_agent.go

@@ -57,6 +57,7 @@ var (
 	// CAP_NET_BIND_SERVICE capability may bind to these port numbers.
 	vSockPort            = 1024
 	kata9pDevType        = "9p"
+	kataMmioBlkDevType   = "mmioblk"
 	kataBlkDevType       = "blk"
 	kataSCSIDevType      = "scsi"
 	sharedDir9pOptions   = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"}
@@ -831,10 +832,15 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
 			ContainerPath: dev.ContainerPath,
 		}
 
-		if d.SCSIAddr == "" {
+		switch c.sandbox.config.HypervisorConfig.BlockDeviceDriver {
+		case config.VirtioMmio:
+			kataDevice.Type = kataMmioBlkDevType
+			kataDevice.Id = d.VirtPath
+			kataDevice.VmPath = d.VirtPath
+		case config.VirtioBlock:
 			kataDevice.Type = kataBlkDevType
 			kataDevice.Id = d.PCIAddr
-		} else {
+		case config.VirtioSCSI:
 			kataDevice.Type = kataSCSIDevType
 			kataDevice.Id = d.SCSIAddr
 		}
@@ -883,7 +889,10 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 			return nil, fmt.Errorf("malformed block drive")
 		}
 
-		if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
+		if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
+			rootfs.Driver = kataMmioBlkDevType
+			rootfs.Source = blockDrive.VirtPath
+		} else if sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
 			rootfs.Driver = kataBlkDevType
 			rootfs.Source = blockDrive.PCIAddr
 		} else {
@@ -1086,9 +1095,12 @@ func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage {
 			k.Logger().Error("malformed block drive")
 			continue
 		}
-		if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
+		if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioBlock {
 			vol.Driver = kataBlkDevType
 			vol.Source = blockDrive.PCIAddr
+		} else if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == config.VirtioMmio {
+			vol.Driver = kataMmioBlkDevType
+			vol.Source = blockDrive.VirtPath
 		} else {
 			vol.Driver = kataSCSIDevType
 			vol.Source = blockDrive.SCSIAddr

virtcontainers/kata_agent_test.go

@@ -402,9 +402,16 @@ func TestAppendDevices(t *testing.T) {
 		},
 	}
 
+	sandboxConfig := &SandboxConfig{
+		HypervisorConfig: HypervisorConfig{
+			BlockDeviceDriver: config.VirtioBlock,
+		},
+	}
+
 	c := &Container{
 		sandbox: &Sandbox{
-			devManager: manager.NewDeviceManager("virtio-scsi", ctrDevices),
+			devManager: manager.NewDeviceManager("virtio-blk", ctrDevices),
+			config:     sandboxConfig,
 		},
 	}
 	c.devices = append(c.devices, ContainerDevice{

virtcontainers/qemu.go

@@ -372,7 +372,7 @@ func (q *qemu) buildDevices(initrdPath string) ([]govmmQemu.Device, *govmmQemu.I
 	}
 
 	var ioThread *govmmQemu.IOThread
-	if q.config.BlockDeviceDriver == VirtioSCSI {
+	if q.config.BlockDeviceDriver == config.VirtioSCSI {
 		devices, ioThread = q.arch.appendSCSIController(devices, q.config.EnableIOThreads)
 	}
 
@@ -746,7 +746,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
 			return err
 		}
 
-		if q.config.BlockDeviceDriver == VirtioBlock {
+		if q.config.BlockDeviceDriver == config.VirtioBlock {
 			driver := "virtio-blk-pci"
 			addr, bridge, err := q.addDeviceToBridge(drive.ID)
 			if err != nil {
@@ -776,7 +776,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
 			}
 		}
 	} else {
-		if q.config.BlockDeviceDriver == VirtioBlock {
+		if q.config.BlockDeviceDriver == config.VirtioBlock {
 			if err := q.removeDeviceFromBridge(drive.ID); err != nil {
 				return err
 			}

virtcontainers/qemu_arch_base.go

@@ -128,14 +128,6 @@ const (
 // 0 is reserved.
 const bridgePCIStartAddr = 2
 
-const (
-	// VirtioBlock means use virtio-blk for hotplugging drives
-	VirtioBlock = "virtio-blk"
-
-	// VirtioSCSI means use virtio-scsi for hotplugging drives
-	VirtioSCSI = "virtio-scsi"
-)
-
 const (
 	// QemuPCLite is the QEMU pc-lite machine type for amd64
 	QemuPCLite = "pc-lite"

virtcontainers/sandbox_test.go

@@ -1553,7 +1553,7 @@ func TestAttachBlockDevice(t *testing.T) {
 	hypervisor := &mockHypervisor{}
 
 	hConfig := HypervisorConfig{
-		BlockDeviceDriver: VirtioBlock,
+		BlockDeviceDriver: config.VirtioBlock,
 	}
 
 	sconfig := &SandboxConfig{
@@ -1598,7 +1598,7 @@ func TestAttachBlockDevice(t *testing.T) {
 		DevType:       "b",
 	}
 
-	dm := manager.NewDeviceManager(VirtioBlock, nil)
+	dm := manager.NewDeviceManager(config.VirtioBlock, nil)
 	device, err := dm.NewDevice(deviceInfo)
 	assert.Nil(t, err)
 	_, ok := device.(*drivers.BlockDevice)
@@ -1618,7 +1618,7 @@ func TestAttachBlockDevice(t *testing.T) {
 	err = device.Detach(sandbox)
 	assert.Nil(t, err)
 
-	container.sandbox.config.HypervisorConfig.BlockDeviceDriver = VirtioSCSI
+	container.sandbox.config.HypervisorConfig.BlockDeviceDriver = config.VirtioSCSI
 	err = device.Attach(sandbox)
 	assert.Nil(t, err)
 
@@ -1638,14 +1638,14 @@ func TestPreAddDevice(t *testing.T) {
 	hypervisor := &mockHypervisor{}
 
 	hConfig := HypervisorConfig{
-		BlockDeviceDriver: VirtioBlock,
+		BlockDeviceDriver: config.VirtioBlock,
 	}
 
 	sconfig := &SandboxConfig{
 		HypervisorConfig: hConfig,
 	}
 
-	dm := manager.NewDeviceManager(VirtioBlock, nil)
+	dm := manager.NewDeviceManager(config.VirtioBlock, nil)
 	// create a sandbox first
 	sandbox := &Sandbox{
 		id:         testSandboxID,