From ed1078c800a0a95d5a106df748c62fa9d5b2ed67 Mon Sep 17 00:00:00 2001 From: Archana Shinde Date: Fri, 30 Mar 2018 17:22:54 -0700 Subject: [PATCH] volumes: Attach volumes that are block device files as block devices Check if a volume passed to the container with -v is a block device file, and if so pass the block device by hotplugging it to the VM instead of passing this as a 9pfs volume. This would give us better performance. Add block device associated with a volume to the list of container devices, so that it is detached with all other devices when the container is stopped with detachDevices() Fixes #137 Signed-off-by: Archana Shinde --- virtcontainers/container.go | 27 ++++++++++ virtcontainers/hyperstart_agent.go | 13 +++++ virtcontainers/kata_agent.go | 83 ++++++++++++++++++++++++++++-- virtcontainers/mount.go | 5 ++ 4 files changed, 125 insertions(+), 3 deletions(-) diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 6bbe7825a5..e085e9b221 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -25,6 +25,7 @@ import ( "time" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) // Process gathers data related to a container process. @@ -311,6 +312,32 @@ func (c *Container) mountSharedDirMounts(hostSharedDir, guestSharedDir string) ( continue } + var stat unix.Stat_t + if err := unix.Stat(m.Source, &stat); err != nil { + return nil, err + } + + // Check if mount is a block device file. If it is, the block device will be attached to the host + // instead of passing this as a shared mount. + if c.checkBlockDeviceSupport() && stat.Mode&unix.S_IFBLK == unix.S_IFBLK { + b := &BlockDevice{ + DeviceType: DeviceBlock, + DeviceInfo: DeviceInfo{ + HostPath: m.Source, + ContainerPath: m.Destination, + DevType: "b", + }, + } + + // Attach this block device, all other devices passed in the config have been attached at this point + if err := b.attach(c.pod.hypervisor, c); err != nil { + return nil, err + } + + c.mounts[idx].BlockDevice = b + continue + } + randBytes, err := generateRandomBytes(8) if err != nil { return nil, err diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index 73b3cac688..73294e3a75 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -413,6 +413,17 @@ func (h *hyper) stopPod(pod Pod) error { return h.proxy.stop(pod, h.state.ProxyPid) } +// handleBlockVolumes handles volumes that are block device files, by +// appending the block device to the list of devices associated with the +// container. +func (h *hyper) handleBlockVolumes(c *Container) { + for _, m := range c.mounts { + if m.BlockDevice != nil { + c.devices = append(c.devices, m.BlockDevice) + } + } +} + func (h *hyper) startOneContainer(pod Pod, c *Container) error { process, err := h.buildHyperContainerProcess(c.config.Cmd) if err != nil { @@ -475,6 +486,8 @@ func (h *hyper) startOneContainer(pod Pod, c *Container) error { fsmap := fsMapFromMounts(newMounts) + h.handleBlockVolumes(c) + // Append container mounts for block devices passed with --device. for _, device := range c.devices { d, ok := device.(*BlockDevice) diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 22cbc42ce2..879e2e9744 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -539,6 +539,35 @@ func (k *kataAgent) replaceOCIMountSource(spec *specs.Spec, guestMounts []Mount) return nil } +func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error { + ociMounts := spec.Mounts + var index int + var m specs.Mount + + for i, v := range volumeStorages { + for index, m = range ociMounts { + if m.Destination != v.MountPoint { + continue + } + + // Create a temporary location to mount the Storage. Mounting to the correct location + // will be handled by the OCI mount structure. + filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) + path := filepath.Join(kataGuestSharedDir, filename) + + k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) + ociMounts[index].Source = path + volumeStorages[i].MountPoint = path + + break + } + if index == len(ociMounts) { + return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint) + } + } + return nil +} + func constraintGRPCSpec(grpcSpec *grpc.Spec) { // Disable Hooks since they have been handled on the host and there is // no reason to send them to the agent. It would make no sense to try @@ -720,6 +749,20 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (p *Process, err err return nil, err } + // Append container devices for block devices passed with --device. + ctrDevices = k.appendDevices(ctrDevices, c.devices) + + // Handle all the volumes that are block device files. + // Note this call modifies the list of container devices to make sure + // all hotplugged devices are unplugged, so this needs be done + // after devices passed with --device are handled. + volumeStorages := k.handleBlockVolumes(c) + if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { + return nil, err + } + + ctrStorages = append(ctrStorages, volumeStorages...) + grpcSpec, err := grpc.OCItoGRPC(ociSpec) if err != nil { return nil, err @@ -732,9 +775,6 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (p *Process, err err // irrelevant information to the agent. constraintGRPCSpec(grpcSpec) - // Append container devices for block devices passed with --device. - ctrDevices = k.appendDevices(ctrDevices, c.devices) - req := &grpc.CreateContainerRequest{ ContainerId: c.id, ExecId: c.id, @@ -760,6 +800,43 @@ func (k *kataAgent) createContainer(pod *Pod, c *Container) (p *Process, err err k.state.URL, c.config.Cmd, createNSList, enterNSList) } +// handleBlockVolumes handles volumes that are block devices files +// by passing the block devices as Storage to the agent. +func (k *kataAgent) handleBlockVolumes(c *Container) []*grpc.Storage { + + var volumeStorages []*grpc.Storage + + for _, m := range c.mounts { + b := m.BlockDevice + + if b == nil { + continue + } + + // Add the block device to the list of container devices, to make sure the + // device is detached with detachDevices() for a container. + c.devices = append(c.devices, b) + + vol := &grpc.Storage{} + + if c.pod.config.HypervisorConfig.BlockDeviceDriver == VirtioBlock { + vol.Driver = kataBlkDevType + vol.Source = b.VirtPath + } else { + vol.Driver = kataSCSIDevType + vol.Source = b.SCSIAddr + } + + vol.MountPoint = b.DeviceInfo.ContainerPath + vol.Fstype = "bind" + vol.Options = []string{"bind"} + + volumeStorages = append(volumeStorages, vol) + } + + return volumeStorages +} + func (k *kataAgent) startContainer(pod Pod, c *Container) error { req := &grpc.StartContainerRequest{ ContainerId: c.id, diff --git a/virtcontainers/mount.go b/virtcontainers/mount.go index 2502ac1e85..9f5b078eba 100644 --- a/virtcontainers/mount.go +++ b/virtcontainers/mount.go @@ -281,6 +281,11 @@ type Mount struct { // ReadOnly specifies if the mount should be read only or not ReadOnly bool + + // BlockDevice represents block device that is attached to the + // VM in case this mount is a block device file or a directory + // backed by a block device. + BlockDevice *BlockDevice } func bindUnmountContainerRootfs(sharedDir, podID, cID string) error {