github/kata-containers
1
0
Fork 1
mirror of https://github.com/kata-containers/kata-containers.git synced 2025-07-02 02:02:24 +00:00
Code Issues Projects Releases Wiki Activity

block: Add new block storage driver "nvdimm"

Browse Source 
Set block_device_driver to "nvdimm" will make the hypervisor use
the block device as NVDIMM disk.

Fixes: #1032

Signed-off-by: Hui Zhu <teawater@hyper.sh>
This commit is contained in:
Hui Zhu 2018-12-14 12:39:04 +08:00
parent b954eecad1
commit ef75c3d19e
7 changed files with 107 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cli/config/configuration.toml.in
View File

@ -91,8 +91,8 @@ default_memory = @DEFMEMSZ@
disable_block_device_use = @DEFDISABLEBLOCK@
# Block storage driver to be used for the hypervisor in case the container
# rootfs is backed by a block device. This is either virtio-scsi or
# virtio-blk.
# rootfs is backed by a block device. This is virtio-scsi, virtio-blk
# or nvdimm.
block_device_driver = "@DEFBLOCKSTORAGEDRIVER@"
# Specifies cache-related options will be set to block devices or not.

2
pkg/katautils/config.go
View File

@ -294,7 +294,7 @@ func (h hypervisor) defaultBridges() uint32 {
}
func (h hypervisor) blockDeviceDriver() (string, error) {
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio}
supportedBlockDrivers := []string{config.VirtioSCSI, config.VirtioBlock, config.VirtioMmio, config.Nvdimm}
if h.BlockDeviceDriver == "" {
return defaultBlockDeviceDriver, nil

6
virtcontainers/device/config/config.go
View File

@ -47,6 +47,9 @@ const (
// VirtioSCSI means use virtio-scsi for hotplugging drives
VirtioSCSI = "virtio-scsi"
// Nvdimm means use nvdimm for hotplugging drives
Nvdimm = "nvdimm"
)
// Defining these as a variable instead of a const, to allow
@ -119,6 +122,9 @@ type BlockDrive struct {
// SCSI address is in the format SCSI-Id:LUN
SCSIAddr string
// NvdimmID is the nvdimm id inside the VM
NvdimmID string
// VirtPath at which the device appears inside the VM, outside of the container mount namespace
VirtPath string
}

4
virtcontainers/device/drivers/block.go
View File

@ -78,7 +78,7 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
}
drive.SCSIAddr = scsiAddr
} else {
} else if customOptions["block-driver"] != "nvdimm" {
var globalIdx int
switch customOptions["block-driver"] {
@ -102,7 +102,7 @@ func (device *BlockDevice) Attach(devReceiver api.DeviceReceiver) (err error) {
drive.VirtPath = filepath.Join("/dev", driveName)
}
deviceLogger().WithField("device", device.DeviceInfo.HostPath).Info("Attaching block device")
deviceLogger().WithField("device", device.DeviceInfo.HostPath).WithField("VirtPath", drive.VirtPath).Infof("Attaching %s device", customOptions["block-driver"])
device.BlockDrive = drive
if err = devReceiver.HotplugAddDevice(device, config.DeviceBlock); err != nil {
return err

4
virtcontainers/device/manager/manager.go
View File

@ -26,6 +26,8 @@ const (
VirtioBlock string = "virtio-blk"
// VirtioSCSI indicates block driver is virtio-scsi based
VirtioSCSI string = "virtio-scsi"
// Nvdimm indicates block driver is nvdimm based
Nvdimm string = "nvdimm"
)
var (
@ -61,6 +63,8 @@ func NewDeviceManager(blockDriver string, devices []api.Device) api.DeviceManage
dm.blockDriver = VirtioMmio
} else if blockDriver == VirtioBlock {
dm.blockDriver = VirtioBlock
} else if blockDriver == Nvdimm {
dm.blockDriver = Nvdimm
} else {
dm.blockDriver = VirtioSCSI
}

4
virtcontainers/kata_agent.go
View File

@ -62,6 +62,7 @@ var (
kataMmioBlkDevType = "mmioblk"
kataBlkDevType = "blk"
kataSCSIDevType = "scsi"
kataNvdimmDevType = "nvdimm"
sharedDir9pOptions = []string{"trans=virtio,version=9p2000.L,cache=mmap", "nodev"}
shmDir = "shm"
kataEphemeralDevType = "ephemeral"
@ -883,6 +884,9 @@ func (k *kataAgent) appendDevices(deviceList []*grpc.Device, c *Container) []*gr
case config.VirtioSCSI:
kataDevice.Type = kataSCSIDevType
kataDevice.Id = d.SCSIAddr
case config.Nvdimm:
kataDevice.Type = kataNvdimmDevType
kataDevice.VmPath = fmt.Sprintf("/dev/pmem%s", d.NvdimmID)
}
deviceList = append(deviceList, kataDevice)

132
virtcontainers/qemu.go
View File

@ -8,20 +8,22 @@ package virtcontainers
import (
"context"
"fmt"
govmmQemu "github.com/intel/govmm/qemu"
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
"github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
"math"
"os"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
govmmQemu "github.com/intel/govmm/qemu"
"github.com/kata-containers/runtime/virtcontainers/pkg/uuid"
opentracing "github.com/opentracing/opentracing-go"
"github.com/sirupsen/logrus"
"unsafe"
"github.com/kata-containers/runtime/virtcontainers/device/config"
"github.com/kata-containers/runtime/virtcontainers/utils"
"golang.org/x/sys/unix"
)
// romFile is the file name of the ROM that can be used for virtio-pci devices.
@ -73,6 +75,8 @@ type qemu struct {
fds []*os.File
ctx context.Context
nvdimmCount int
}
const (
@ -221,6 +225,20 @@ func (q *qemu) init(ctx context.Context, id string, hypervisorConfig *Hypervisor
q.config = *hypervisorConfig
q.arch = newQemuArch(q.config)
initrdPath, err := q.config.InitrdAssetPath()
if err != nil {
return err
}
imagePath, err := q.config.ImageAssetPath()
if err != nil {
return err
}
if initrdPath == "" && imagePath != "" {
q.nvdimmCount = 1
} else {
q.nvdimmCount = 0
}
if err = q.storage.fetchHypervisorState(q.id, &q.state); err != nil {
q.Logger().Debug("Creating bridges")
q.state.Bridges = q.arch.bridges(q.config.DefaultBridges)
@ -727,6 +745,69 @@ func (q *qemu) removeDeviceFromBridge(ID string) error {
return err
}
func (q *qemu) hotplugAddBlockDevice(drive *config.BlockDrive, op operation, devID string) error {
var err error
if q.config.BlockDeviceDriver == config.Nvdimm {
var blocksize int64
file, err := os.Open(drive.File)
if err != nil {
return err
}
if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, file.Fd(), unix.BLKGETSIZE64, uintptr(unsafe.Pointer(&blocksize))); err != 0 {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteNVDIMMDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, drive.File, blocksize); err != nil {
q.Logger().WithError(err).Errorf("Failed to add NVDIMM device %s", drive.File)
return err
}
drive.NvdimmID = strconv.Itoa(q.nvdimmCount)
q.nvdimmCount++
return nil
}
if q.config.BlockDeviceCacheSet {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush)
} else {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID)
}
if err != nil {
return err
}
if q.config.BlockDeviceDriver == config.VirtioBlock {
driver := "virtio-blk-pci"
addr, bridge, err := q.addDeviceToBridge(drive.ID)
if err != nil {
return err
}
// PCI address is in the format bridge-addr/device-addr eg. "03/02"
drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, true, q.arch.runNested()); err != nil {
return err
}
} else {
driver := "scsi-hd"
// Bus exposed by the SCSI Controller
bus := scsiControllerID + ".0"
// Get SCSI-id and LUN based on the order of attaching drives.
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, q.arch.runNested()); err != nil {
return err
}
}
return nil
}
func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error {
err := q.qmpSetup()
if err != nil {
@ -736,44 +817,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
devID := "virtio-" + drive.ID
if op == addDevice {
if q.config.BlockDeviceCacheSet {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAddWithCache(q.qmpMonitorCh.ctx, drive.File, drive.ID, q.config.BlockDeviceCacheDirect, q.config.BlockDeviceCacheNoflush)
} else {
err = q.qmpMonitorCh.qmp.ExecuteBlockdevAdd(q.qmpMonitorCh.ctx, drive.File, drive.ID)
}
if err != nil {
return err
}
if q.config.BlockDeviceDriver == config.VirtioBlock {
driver := "virtio-blk-pci"
addr, bridge, err := q.addDeviceToBridge(drive.ID)
if err != nil {
return err
}
// PCI address is in the format bridge-addr/device-addr eg. "03/02"
drive.PCIAddr = fmt.Sprintf("%02x", bridge.Addr) + "/" + addr
if err = q.qmpMonitorCh.qmp.ExecutePCIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, addr, bridge.ID, romFile, true, q.arch.runNested()); err != nil {
return err
}
} else {
driver := "scsi-hd"
// Bus exposed by the SCSI Controller
bus := scsiControllerID + ".0"
// Get SCSI-id and LUN based on the order of attaching drives.
scsiID, lun, err := utils.GetSCSIIdLun(drive.Index)
if err != nil {
return err
}
if err = q.qmpMonitorCh.qmp.ExecuteSCSIDeviceAdd(q.qmpMonitorCh.ctx, drive.ID, devID, driver, bus, romFile, scsiID, lun, true, q.arch.runNested()); err != nil {
return err
}
}
err = q.hotplugAddBlockDevice(drive, op, devID)
} else {
if q.config.BlockDeviceDriver == config.VirtioBlock {
if err := q.removeDeviceFromBridge(drive.ID); err != nil {
@ -790,7 +834,7 @@ func (q *qemu) hotplugBlockDevice(drive *config.BlockDrive, op operation) error
}
}
return nil
return err
}
func (q *qemu) hotplugVFIODevice(device *config.VFIODev, op operation) error {