Merge pull request #267 from amshinde/pass-pci-addr

Determine and pass PCI addresses for block devices

virtcontainers/container.go

@@ -212,6 +212,17 @@ func (c *Container) setStateHotpluggedDrive(hotplugged bool) error {
 	return nil
 }
 
+func (c *Container) setContainerRootfsPCIAddr(addr string) error {
+	c.state.RootfsPCIAddr = addr
+
+	err := c.sandbox.storage.storeContainerResource(c.sandbox.id, c.id, stateFileType, c.state)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // GetAnnotations returns container's annotations
 func (c *Container) GetAnnotations() map[string]string {
 	return c.config.Annotations
@@ -788,9 +799,14 @@ func (c *Container) hotplugDrive() error {
 		Index:  driveIndex,
 	}
 
-	if err := c.sandbox.hypervisor.hotplugAddDevice(drive, blockDev); err != nil {
+	if err := c.sandbox.hypervisor.hotplugAddDevice(&drive, blockDev); err != nil {
 		return err
 	}
+
+	if drive.PCIAddr != "" {
+		c.setContainerRootfsPCIAddr(drive.PCIAddr)
+	}
+
 	c.setStateHotpluggedDrive(true)
 
 	if err := c.setStateBlockIndex(driveIndex); err != nil {
@@ -813,7 +829,7 @@ func (c *Container) removeDrive() (err error) {
 		c.Logger().Info("unplugging block device")
 
 		devID := makeNameID("drive", c.id)
-		drive := Drive{
+		drive := &Drive{
 			ID: devID,
 		}
 

virtcontainers/device.go

@@ -334,6 +334,9 @@ type BlockDevice struct {
 
 	// Path at which the device appears inside the VM, outside of the container mount namespace
 	VirtPath string
+
+	// PCI Slot of the block device
+	PCIAddr string
 }
 
 func newBlockDevice(devInfo DeviceInfo) *BlockDevice {
@@ -380,7 +383,7 @@ func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) {
 
 	deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
 
-	if err = h.hotplugAddDevice(drive, blockDev); err != nil {
+	if err = h.hotplugAddDevice(&drive, blockDev); err != nil {
 		return err
 	}
 
@@ -388,6 +391,7 @@ func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) {
 
 	if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
 		device.VirtPath = filepath.Join("/dev", driveName)
+		device.PCIAddr = drive.PCIAddr
 	} else {
 		scsiAddr, err := getSCSIAddress(index)
 		if err != nil {
@@ -404,7 +408,7 @@ func (device BlockDevice) detach(h hypervisor) error {
 	if device.DeviceInfo.Hotplugged {
 		deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
 
-		drive := Drive{
+		drive := &Drive{
 			ID: makeNameID("drive", device.DeviceInfo.ID),
 		}
 

virtcontainers/kata_agent.go

@@ -40,7 +40,6 @@ var (
 	kataGuestSharedDir          = "/run/kata-containers/shared/containers/"
 	mountGuest9pTag             = "kataShared"
 	type9pFs                    = "9p"
-	devPath                     = "/dev"
 	vsockSocketScheme           = "vsock"
 	kata9pDevType               = "9p"
 	kataBlkDevType              = "blk"
@@ -629,7 +628,7 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []Device) [
 
 		if d.SCSIAddr == "" {
 			kataDevice.Type = kataBlkDevType
-			kataDevice.VmPath = d.VirtPath
+			kataDevice.Id = d.PCIAddr
 		} else {
 			kataDevice.Type = kataSCSIDevType
 			kataDevice.Id = d.SCSIAddr
@@ -692,15 +691,8 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
 		// If virtio-scsi driver, the agent will be able to find the
 		// device based on the provided address.
 		if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
-			// driveName is the predicted virtio-block guest name (the vd* in /dev/vd*).
-			driveName, err := getVirtDriveName(c.state.BlockIndex)
-			if err != nil {
-				return nil, err
-			}
-			virtPath := filepath.Join(devPath, driveName)
-
 			rootfs.Driver = kataBlkDevType
-			rootfs.Source = virtPath
+			rootfs.Source = c.state.RootfsPCIAddr
 		} else {
 			scsiAddr, err := getSCSIAddress(c.state.BlockIndex)
 			if err != nil {

virtcontainers/kata_agent_test.go

@@ -25,8 +25,8 @@ import (
 
 var (
 	testKataProxyURLTempl  = "unix://%s/kata-proxy-test.sock"
-	testBlockDeviceVirtPath = "testBlockDeviceVirtPath"
 	testBlockDeviceCtrPath = "testBlockDeviceCtrPath"
+	testPCIAddr            = "04/02"
 )
 
 func proxyHandlerDiscard(c net.Conn) {
@@ -366,16 +366,16 @@ func TestAppendDevices(t *testing.T) {
 	expected := []*pb.Device{
 		{
 			Type:          kataBlkDevType,
-			VmPath:        testBlockDeviceVirtPath,
 			ContainerPath: testBlockDeviceCtrPath,
+			Id:            testPCIAddr,
 		},
 	}
 	ctrDevices := []Device{
 		&BlockDevice{
-			VirtPath: testBlockDeviceVirtPath,
 			DeviceInfo: DeviceInfo{
 				ContainerPath: testBlockDeviceCtrPath,
 			},
+			PCIAddr: testPCIAddr,
 		},
 	}
 

virtcontainers/qemu.go

@@ -565,7 +565,7 @@ func (q *qemu) qmpSetup() (*govmmQemu.QMP, error) {
 	return qmp, nil
 }
 
-func (q *qemu) addDeviceToBridge(ID string) (string, string, error) {
+func (q *qemu) addDeviceToBridge(ID string) (string, Bridge, error) {
 	var err error
 	var addr uint32
 
@@ -573,11 +573,11 @@ func (q *qemu) addDeviceToBridge(ID string) (string, string, error) {
 	for _, b := range q.state.Bridges {
 		addr, err = b.addDevice(ID)
 		if err == nil {
-			return fmt.Sprintf("0x%x", addr), b.ID, nil
+			return fmt.Sprintf("%02x", addr), b, nil
 		}
 	}
 
-	return "", "", err
+	return "", Bridge{}, err
 }
 
 func (q *qemu) removeDeviceFromBridge(ID string) error {
@@ -593,7 +593,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error {
 	return err
 }
 
-func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error {
+func (q *qemu) hotplugBlockDevice(drive *Drive, op operation) error {
 	defer func(qemu *qemu) {
 		if q.qmpMonitorCh.qmp != nil {
 			q.qmpMonitorCh.qmp.Shutdown()
@@ -616,12 +616,15 @@ func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error {
 
 		if q.config.BlockDeviceDriver == VirtioBlock {
 			driver := "virtio-blk-pci"
-			addr, bus, err := q.addDeviceToBridge(drive.ID)
+			addr, bridge, err := q.addDeviceToBridge(drive.ID)
 			if err != nil {
 				return err
 			}
 
-			if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus); err != nil {
+			// PCI address is in the format bridge-addr/device-addr eg. "03/02"
+			drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
+
+			if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID); err != nil {
 				return err
 			}
 		} else {
@@ -676,12 +679,12 @@ func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error {
 	devID := "vfio-" + device.DeviceInfo.ID
 
 	if op == addDevice {
-		addr, bus, err := q.addDeviceToBridge(devID)
+		addr, bridge, err := q.addDeviceToBridge(devID)
 		if err != nil {
 			return err
 		}
 
-		if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus); err != nil {
+		if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID); err != nil {
 			return err
 		}
 	} else {
@@ -700,7 +703,7 @@ func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error {
 func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) error {
 	switch devType {
 	case blockDev:
-		drive := devInfo.(Drive)
+		drive := devInfo.(*Drive)
 		return q.hotplugBlockDevice(drive, op)
 	case cpuDev:
 		vcpus := devInfo.(uint32)

virtcontainers/qemu_amd64.go

@@ -205,7 +205,7 @@ func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []Bridge)
 			t = govmmQemu.PCIEBridge
 		}
 
-		b.Addr = bridgePCIStartAddr + idx
+		bridges[idx].Addr = bridgePCIStartAddr + idx
 
 		devices = append(devices,
 			govmmQemu.BridgeDevice{
@@ -215,7 +215,7 @@ func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []Bridge)
 				// Each bridge is required to be assigned a unique chassis id > 0
 				Chassis: (idx + 1),
 				SHPC:    true,
-				Addr:    strconv.FormatInt(int64(b.Addr), 10),
+				Addr:    strconv.FormatInt(int64(bridges[idx].Addr), 10),
 			},
 		)
 	}

virtcontainers/qemu_arch_base.go

@@ -328,7 +328,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []Bridg
 			t = govmmQemu.PCIEBridge
 		}
 
-		b.Addr = bridgePCIStartAddr + idx
+		bridges[idx].Addr = bridgePCIStartAddr + idx
 
 		devices = append(devices,
 			govmmQemu.BridgeDevice{
@@ -338,7 +338,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []Bridg
 				// Each bridge is required to be assigned a unique chassis id > 0
 				Chassis: (idx + 1),
 				SHPC:    true,
-				Addr:    strconv.FormatInt(int64(b.Addr), 10),
+				Addr:    strconv.FormatInt(int64(bridges[idx].Addr), 10),
 			},
 		)
 	}

virtcontainers/sandbox.go

@@ -60,6 +60,9 @@ type State struct {
 
 	// Bool to indicate if the drive for a container was hotplugged.
 	HotpluggedDrive bool `json:"hotpluggedDrive"`
+
+	// PCI slot at which the block device backing the container rootfs is attached.
+	RootfsPCIAddr string `json:"rootfsPCIAddr"`
 }
 
 // valid checks that the sandbox state is valid.
@@ -237,6 +240,9 @@ type Drive struct {
 
 	// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
 	Index int
+
+	// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
+	PCIAddr string
 }
 
 // EnvVar is a key/value structure representing a command