From c39281ad65f853e791c36a11092679c8b6dea871 Mon Sep 17 00:00:00 2001 From: Feng Wang Date: Fri, 14 Jan 2022 13:21:41 -0800 Subject: [PATCH] runtime: update container creation to work with direct assigned volumes During the container creation, it will parse the mount info file of the direct assigned volumes and update the in memory mount object. Fixes: #3454 Signed-off-by: Feng Wang --- src/runtime/virtcontainers/container.go | 23 ++++ src/runtime/virtcontainers/kata_agent.go | 124 +++++++++--------- src/runtime/virtcontainers/kata_agent_test.go | 18 ++- src/runtime/virtcontainers/mount.go | 11 +- 4 files changed, 106 insertions(+), 70 deletions(-) diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 82aae79e37..1a4cb8e839 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" @@ -611,6 +612,28 @@ func (c *Container) createBlockDevices(ctx context.Context) error { continue } + // Handle directly assigned volume. Update the mount info based on the mount info json. + mntInfo, e := volume.VolumeMountInfo(m.Source) + if e != nil && !os.IsNotExist(e) { + c.Logger().WithError(e).WithField("mount-source", m.Source). + Error("failed to parse the mount info file for a direct assigned volume") + continue + } + + if mntInfo != nil { + // Write out sandbox info file on the mount source to allow CSI to communicate with the runtime + if err := volume.RecordSandboxId(c.sandboxID, m.Source); err != nil { + c.Logger().WithError(err).Error("error writing sandbox info") + } + + c.mounts[i].Source = mntInfo.Device + c.mounts[i].Type = mntInfo.FsType + c.mounts[i].Options = mntInfo.Options + m.Source = mntInfo.Device + m.Type = mntInfo.FsType + m.Options = mntInfo.Options + } + var stat unix.Stat_t if err := unix.Stat(m.Source, &stat); err != nil { return fmt.Errorf("stat %q failed: %v", m.Source, err) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 47a1c1cc3f..71781e4ade 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -882,35 +882,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st return nil } -func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error { - ociMounts := spec.Mounts - var index int - var m specs.Mount - - for i, v := range volumeStorages { - for index, m = range ociMounts { - if m.Destination != v.MountPoint { - continue - } - - // Create a temporary location to mount the Storage. Mounting to the correct location - // will be handled by the OCI mount structure. - filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) - path := filepath.Join(kataGuestSandboxStorageDir(), filename) - - k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) - ociMounts[index].Source = path - volumeStorages[i].MountPoint = path - - break - } - if index == len(ociMounts) { - return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint) - } - } - return nil -} - func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) { // Disable Hooks since they have been handled on the host and there is // no reason to send them to the agent. It would make no sense to try @@ -1247,19 +1218,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co // Append container devices for block devices passed with --device. ctrDevices = k.appendDevices(ctrDevices, c) - // Handle all the volumes that are block device files. - // Note this call modifies the list of container devices to make sure - // all hotplugged devices are unplugged, so this needs be done - // after devices passed with --device are handled. - volumeStorages, err := k.handleBlockVolumes(c) + // Block based volumes will require some adjustments in the OCI spec, and creation of + // storage objects to pass to the agent. + volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec) if err != nil { return nil, err } - if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { - return nil, err - } - ctrStorages = append(ctrStorages, volumeStorages...) grpcSpec, err := grpc.OCItoGRPC(ociSpec) @@ -1522,16 +1487,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D vol.Options = []string{"bind"} vol.MountPoint = m.Destination + // Assign the type from the mount, if it's specified (e.g. direct assigned volume) + if m.Type != "" { + vol.Fstype = m.Type + vol.Options = m.Options + } + return vol, nil } -// handleBlockVolumes handles volumes that are block devices files -// by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { +func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) { + var vol *grpc.Storage + + id := m.BlockDeviceID + device := c.sandbox.devManager.GetDeviceByID(id) + if device == nil { + k.Logger().WithField("device", id).Error("failed to find device by id") + return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) + } + + var err error + switch device.DeviceType() { + case config.DeviceBlock: + vol, err = k.handleDeviceBlockVolume(c, m, device) + case config.VhostUserBlk: + vol, err = k.handleVhostUserBlkVolume(c, m, device) + default: + return nil, fmt.Errorf("Unknown device type") + } + + return vol, err +} + +// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the +// given container and will update the OCI spec to utilize this mount point as the new source for the +// container volume. The container mount structure is updated to store the guest destination mountpoint. +func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage - for _, m := range c.mounts { + for i, m := range c.mounts { id := m.BlockDeviceID if len(id) == 0 { @@ -1542,29 +1537,36 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { // device is detached with detachDevices() for a container. c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination}) - var vol *grpc.Storage - - device := c.sandbox.devManager.GetDeviceByID(id) - if device == nil { - k.Logger().WithField("device", id).Error("failed to find device by id") - return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) - } - - var err error - switch device.DeviceType() { - case config.DeviceBlock: - vol, err = k.handleDeviceBlockVolume(c, m, device) - case config.VhostUserBlk: - vol, err = k.handleVhostUserBlkVolume(c, m, device) - default: - k.Logger().Error("Unknown device type") - continue - } - + // Create Storage structure + vol, err := k.createBlkStorageObject(c, m) if vol == nil || err != nil { return nil, err } + // The device will be mounted at a unique location within the VM. Mounting + // to the container specific location is handled within the OCI spec. Let's ensure that + // the storage mount point is unique, and that this is utilized as the source in the OCI + // spec. + filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(vol.MountPoint)) + path := filepath.Join(kataGuestSandboxStorageDir(), filename) + + // Update applicable OCI mount source + for idx, ociMount := range spec.Mounts { + if ociMount.Destination != vol.MountPoint { + continue + } + k.Logger().WithFields(logrus.Fields{ + "original-source": ociMount.Source, + "new-source": path, + }).Debug("Replacing OCI mount source") + spec.Mounts[idx].Source = path + break + } + + // Update storage mountpoint, and save guest device mount path to container mount struct: + vol.MountPoint = path + c.mounts[i].GuestDeviceMount = path + volumeStorages = append(volumeStorages, vol) } diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 16dd9dcfe1..9f861e932b 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -398,24 +398,28 @@ func TestHandleBlockVolume(t *testing.T) { containers[c.id].sandbox = &sandbox containers[c.id].mounts = mounts - volumeStorages, err := k.handleBlockVolumes(c) + vStorage, err := k.createBlkStorageObject(c, vMount) + assert.Nil(t, err, "Error while handling block volumes") + bStorage, err := k.createBlkStorageObject(c, bMount) + assert.Nil(t, err, "Error while handling block volumes") + dStorage, err := k.createBlkStorageObject(c, dMount) assert.Nil(t, err, "Error while handling block volumes") - vStorage := &pb.Storage{ + vStorageExpected := &pb.Storage{ MountPoint: vDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: vPCIPath.String(), } - bStorage := &pb.Storage{ + bStorageExpected := &pb.Storage{ MountPoint: bDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: bPCIPath.String(), } - dStorage := &pb.Storage{ + dStorageExpected := &pb.Storage{ MountPoint: dDestination, Fstype: "ext4", Options: []string{"ro"}, @@ -423,9 +427,9 @@ func TestHandleBlockVolume(t *testing.T) { Source: dPCIPath.String(), } - assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume") - assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume") - assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume") + assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume") + assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume") + assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume") } func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index c2879f2135..0e83bc6899 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) { }, nil } - // We get the mount point by recursively peforming stat on the path + // We get the mount point by recursively performing stat on the path // The point where the device changes indicates the mountpoint for { if mountPoint == "/" { @@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string // Mount describes a container mount. type Mount struct { - Source string + // Source is the source of the mount. + Source string + // Destination is the destination of the mount (within the container). Destination string // Type specifies the type of filesystem to mount. @@ -335,6 +337,11 @@ type Mount struct { // HostPath used to store host side bind mount path HostPath string + // GuestDeviceMount represents the path within the VM that the device + // is mounted. Only relevant for block devices. This is tracked in the event + // runtime wants to query the agent for mount stats. + GuestDeviceMount string + // BlockDeviceID represents block device that is attached to the // VM in case this mount is a block device file or a directory // backed by a block device.