github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-17 08:53:26 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #267 from amshinde/pass-pci-addr

Browse Source 
Determine and pass PCI addresses for block devices
This commit is contained in:
Eric Ernst 2018-05-03 16:20:47 -07:00 committed by GitHub
commit 992c895eaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 53 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
virtcontainers/container.go
View File

@ -212,6 +212,17 @@ func (c *Container) setStateHotpluggedDrive(hotplugged bool) error {
return nil
}
func (c *Container) setContainerRootfsPCIAddr(addr string) error {
c.state.RootfsPCIAddr = addr
err := c.sandbox.storage.storeContainerResource(c.sandbox.id, c.id, stateFileType, c.state)
if err != nil {
return err
}
return nil
}
// GetAnnotations returns container's annotations
func (c *Container) GetAnnotations() map[string]string {
return c.config.Annotations
@ -788,9 +799,14 @@ func (c *Container) hotplugDrive() error {
Index: driveIndex,
}
if err := c.sandbox.hypervisor.hotplugAddDevice(drive, blockDev); err != nil {
if err := c.sandbox.hypervisor.hotplugAddDevice(&drive, blockDev); err != nil {
return err
}
if drive.PCIAddr != "" {
c.setContainerRootfsPCIAddr(drive.PCIAddr)
}
c.setStateHotpluggedDrive(true)
if err := c.setStateBlockIndex(driveIndex); err != nil {
@ -813,7 +829,7 @@ func (c *Container) removeDrive() (err error) {
c.Logger().Info("unplugging block device")
devID := makeNameID("drive", c.id)
drive := Drive{
drive := &Drive{
ID: devID,
}

8
virtcontainers/device.go
View File

@ -334,6 +334,9 @@ type BlockDevice struct {
// Path at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
// PCI Slot of the block device
PCIAddr string
}
func newBlockDevice(devInfo DeviceInfo) *BlockDevice {
@ -380,7 +383,7 @@ func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) {
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
if err = h.hotplugAddDevice(drive, blockDev); err != nil {
if err = h.hotplugAddDevice(&drive, blockDev); err != nil {
return err
}
@ -388,6 +391,7 @@ func (device *BlockDevice) attach(h hypervisor, c *Container) (err error) {
if c.sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
device.VirtPath = filepath.Join("/dev", driveName)
device.PCIAddr = drive.PCIAddr
} else {
scsiAddr, err := getSCSIAddress(index)
if err != nil {
@ -404,7 +408,7 @@ func (device BlockDevice) detach(h hypervisor) error {
if device.DeviceInfo.Hotplugged {
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Unplugging block device")
drive := Drive{
drive := &Drive{
ID: makeNameID("drive", device.DeviceInfo.ID),
}

12
virtcontainers/kata_agent.go
View File

@ -40,7 +40,6 @@ var (
kataGuestSharedDir = "/run/kata-containers/shared/containers/"
mountGuest9pTag = "kataShared"
type9pFs = "9p"
devPath = "/dev"
vsockSocketScheme = "vsock"
kata9pDevType = "9p"
kataBlkDevType = "blk"
@ -629,7 +628,7 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, devices []Device) [
if d.SCSIAddr == "" {
kataDevice.Type = kataBlkDevType
kataDevice.VmPath = d.VirtPath
kataDevice.Id = d.PCIAddr
} else {
kataDevice.Type = kataSCSIDevType
kataDevice.Id = d.SCSIAddr
@ -692,15 +691,8 @@ func (k *kataAgent) createContainer(sandbox *Sandbox, c *Container) (p *Process,
// If virtio-scsi driver, the agent will be able to find the
// device based on the provided address.
if sandbox.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock {
// driveName is the predicted virtio-block guest name (the vd* in /dev/vd*).
driveName, err := getVirtDriveName(c.state.BlockIndex)
if err != nil {
return nil, err
}
virtPath := filepath.Join(devPath, driveName)
rootfs.Driver = kataBlkDevType
rootfs.Source = virtPath
rootfs.Source = c.state.RootfsPCIAddr
} else {
scsiAddr, err := getSCSIAddress(c.state.BlockIndex)
if err != nil {

10
virtcontainers/kata_agent_test.go
View File

@ -24,9 +24,9 @@ import (
)
var (
testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock"
testBlockDeviceVirtPath = "testBlockDeviceVirtPath"
testBlockDeviceCtrPath = "testBlockDeviceCtrPath"
testKataProxyURLTempl = "unix://%s/kata-proxy-test.sock"
testBlockDeviceCtrPath = "testBlockDeviceCtrPath"
testPCIAddr = "04/02"
)
func proxyHandlerDiscard(c net.Conn) {
@ -366,16 +366,16 @@ func TestAppendDevices(t *testing.T) {
expected := []*pb.Device{
{
Type: kataBlkDevType,
VmPath: testBlockDeviceVirtPath,
ContainerPath: testBlockDeviceCtrPath,
Id: testPCIAddr,
},
}
ctrDevices := []Device{
&BlockDevice{
VirtPath: testBlockDeviceVirtPath,
DeviceInfo: DeviceInfo{
ContainerPath: testBlockDeviceCtrPath,
},
PCIAddr: testPCIAddr,
},
}

21
virtcontainers/qemu.go
View File

@ -565,7 +565,7 @@ func (q *qemu) qmpSetup() (*govmmQemu.QMP, error) {
return qmp, nil
}
func (q *qemu) addDeviceToBridge(ID string) (string, string, error) {
func (q *qemu) addDeviceToBridge(ID string) (string, Bridge, error) {
var err error
var addr uint32
@ -573,11 +573,11 @@ func (q *qemu) addDeviceToBridge(ID string) (string, string, error) {
for _, b := range q.state.Bridges {
addr, err = b.addDevice(ID)
if err == nil {
return fmt.Sprintf("0x%x", addr), b.ID, nil
return fmt.Sprintf("%02x", addr), b, nil
}
}
return "", "", err
return "", Bridge{}, err
}
func (q *qemu) removeDeviceFromBridge(ID string) error {
@ -593,7 +593,7 @@ func (q *qemu) removeDeviceFromBridge(ID string) error {
return err
}
func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error {
func (q *qemu) hotplugBlockDevice(drive *Drive, op operation) error {
defer func(qemu *qemu) {
if q.qmpMonitorCh.qmp != nil {
q.qmpMonitorCh.qmp.Shutdown()
@ -616,12 +616,15 @@ func (q *qemu) hotplugBlockDevice(drive Drive, op operation) error {
if q.config.BlockDeviceDriver == VirtioBlock {
driver := "virtio-blk-pci"
addr, bus, err := q.addDeviceToBridge(drive.ID)
addr, bridge, err := q.addDeviceToBridge(drive.ID)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bus); err != nil {
// PCI address is in the format bridge-addr/device-addr eg. "03/02"
drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID); err != nil {
return err
}
} else {
@ -676,12 +679,12 @@ func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error {
devID := "vfio-" + device.DeviceInfo.ID
if op == addDevice {
addr, bus, err := q.addDeviceToBridge(devID)
addr, bridge, err := q.addDeviceToBridge(devID)
if err != nil {
return err
}
if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bus); err != nil {
if err := q.qmpMonitorCh.qmp.ExecutePCIVFIODeviceAdd(q.qmpMonitorCh.ctx, devID, device.BDF, addr, bridge.ID); err != nil {
return err
}
} else {
@ -700,7 +703,7 @@ func (q *qemu) hotplugVFIODevice(device VFIODevice, op operation) error {
func (q *qemu) hotplugDevice(devInfo interface{}, devType deviceType, op operation) error {
switch devType {
case blockDev:
drive := devInfo.(Drive)
drive := devInfo.(*Drive)
return q.hotplugBlockDevice(drive, op)
case cpuDev:
vcpus := devInfo.(uint32)

4
virtcontainers/qemu_amd64.go
View File

@ -205,7 +205,7 @@ func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []Bridge)
t = govmmQemu.PCIEBridge
}
b.Addr = bridgePCIStartAddr + idx
bridges[idx].Addr = bridgePCIStartAddr + idx
devices = append(devices,
govmmQemu.BridgeDevice{
@ -215,7 +215,7 @@ func (q *qemuAmd64) appendBridges(devices []govmmQemu.Device, bridges []Bridge)
// Each bridge is required to be assigned a unique chassis id > 0
Chassis: (idx + 1),
SHPC: true,
Addr: strconv.FormatInt(int64(b.Addr), 10),
Addr: strconv.FormatInt(int64(bridges[idx].Addr), 10),
},
)
}

4
virtcontainers/qemu_arch_base.go
View File

@ -328,7 +328,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []Bridg
t = govmmQemu.PCIEBridge
}
b.Addr = bridgePCIStartAddr + idx
bridges[idx].Addr = bridgePCIStartAddr + idx
devices = append(devices,
govmmQemu.BridgeDevice{
@ -338,7 +338,7 @@ func (q *qemuArchBase) appendBridges(devices []govmmQemu.Device, bridges []Bridg
// Each bridge is required to be assigned a unique chassis id > 0
Chassis: (idx + 1),
SHPC: true,
Addr: strconv.FormatInt(int64(b.Addr), 10),
Addr: strconv.FormatInt(int64(bridges[idx].Addr), 10),
},
)
}

6
virtcontainers/sandbox.go
View File

@ -60,6 +60,9 @@ type State struct {
// Bool to indicate if the drive for a container was hotplugged.
HotpluggedDrive bool `json:"hotpluggedDrive"`
// PCI slot at which the block device backing the container rootfs is attached.
RootfsPCIAddr string `json:"rootfsPCIAddr"`
}
// valid checks that the sandbox state is valid.
@ -237,6 +240,9 @@ type Drive struct {
// Index assigned to the drive. In case of virtio-scsi, this is used as SCSI LUN index
Index int
// PCIAddr is the PCI address used to identify the slot at which the drive is attached.
PCIAddr string
}
// EnvVar is a key/value structure representing a command