diff --git a/src/runtime/virtcontainers/container.go b/src/runtime/virtcontainers/container.go index 82aae79e37..1a4cb8e839 100644 --- a/src/runtime/virtcontainers/container.go +++ b/src/runtime/virtcontainers/container.go @@ -15,6 +15,7 @@ import ( "syscall" "time" + volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume" "github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config" "github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager" @@ -611,6 +612,28 @@ func (c *Container) createBlockDevices(ctx context.Context) error { continue } + // Handle directly assigned volume. Update the mount info based on the mount info json. + mntInfo, e := volume.VolumeMountInfo(m.Source) + if e != nil && !os.IsNotExist(e) { + c.Logger().WithError(e).WithField("mount-source", m.Source). + Error("failed to parse the mount info file for a direct assigned volume") + continue + } + + if mntInfo != nil { + // Write out sandbox info file on the mount source to allow CSI to communicate with the runtime + if err := volume.RecordSandboxId(c.sandboxID, m.Source); err != nil { + c.Logger().WithError(err).Error("error writing sandbox info") + } + + c.mounts[i].Source = mntInfo.Device + c.mounts[i].Type = mntInfo.FsType + c.mounts[i].Options = mntInfo.Options + m.Source = mntInfo.Device + m.Type = mntInfo.FsType + m.Options = mntInfo.Options + } + var stat unix.Stat_t if err := unix.Stat(m.Source, &stat); err != nil { return fmt.Errorf("stat %q failed: %v", m.Source, err) diff --git a/src/runtime/virtcontainers/kata_agent.go b/src/runtime/virtcontainers/kata_agent.go index 47a1c1cc3f..71781e4ade 100644 --- a/src/runtime/virtcontainers/kata_agent.go +++ b/src/runtime/virtcontainers/kata_agent.go @@ -882,35 +882,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st return nil } -func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error { - ociMounts := spec.Mounts - var index int - var m specs.Mount - - for i, v := range volumeStorages { - for index, m = range ociMounts { - if m.Destination != v.MountPoint { - continue - } - - // Create a temporary location to mount the Storage. Mounting to the correct location - // will be handled by the OCI mount structure. - filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination)) - path := filepath.Join(kataGuestSandboxStorageDir(), filename) - - k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path) - ociMounts[index].Source = path - volumeStorages[i].MountPoint = path - - break - } - if index == len(ociMounts) { - return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint) - } - } - return nil -} - func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) { // Disable Hooks since they have been handled on the host and there is // no reason to send them to the agent. It would make no sense to try @@ -1247,19 +1218,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co // Append container devices for block devices passed with --device. ctrDevices = k.appendDevices(ctrDevices, c) - // Handle all the volumes that are block device files. - // Note this call modifies the list of container devices to make sure - // all hotplugged devices are unplugged, so this needs be done - // after devices passed with --device are handled. - volumeStorages, err := k.handleBlockVolumes(c) + // Block based volumes will require some adjustments in the OCI spec, and creation of + // storage objects to pass to the agent. + volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec) if err != nil { return nil, err } - if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil { - return nil, err - } - ctrStorages = append(ctrStorages, volumeStorages...) grpcSpec, err := grpc.OCItoGRPC(ociSpec) @@ -1522,16 +1487,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D vol.Options = []string{"bind"} vol.MountPoint = m.Destination + // Assign the type from the mount, if it's specified (e.g. direct assigned volume) + if m.Type != "" { + vol.Fstype = m.Type + vol.Options = m.Options + } + return vol, nil } -// handleBlockVolumes handles volumes that are block devices files -// by passing the block devices as Storage to the agent. -func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { +func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) { + var vol *grpc.Storage + + id := m.BlockDeviceID + device := c.sandbox.devManager.GetDeviceByID(id) + if device == nil { + k.Logger().WithField("device", id).Error("failed to find device by id") + return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) + } + + var err error + switch device.DeviceType() { + case config.DeviceBlock: + vol, err = k.handleDeviceBlockVolume(c, m, device) + case config.VhostUserBlk: + vol, err = k.handleVhostUserBlkVolume(c, m, device) + default: + return nil, fmt.Errorf("Unknown device type") + } + + return vol, err +} + +// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the +// given container and will update the OCI spec to utilize this mount point as the new source for the +// container volume. The container mount structure is updated to store the guest destination mountpoint. +func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) { var volumeStorages []*grpc.Storage - for _, m := range c.mounts { + for i, m := range c.mounts { id := m.BlockDeviceID if len(id) == 0 { @@ -1542,29 +1537,36 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) { // device is detached with detachDevices() for a container. c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination}) - var vol *grpc.Storage - - device := c.sandbox.devManager.GetDeviceByID(id) - if device == nil { - k.Logger().WithField("device", id).Error("failed to find device by id") - return nil, fmt.Errorf("Failed to find device by id (id=%s)", id) - } - - var err error - switch device.DeviceType() { - case config.DeviceBlock: - vol, err = k.handleDeviceBlockVolume(c, m, device) - case config.VhostUserBlk: - vol, err = k.handleVhostUserBlkVolume(c, m, device) - default: - k.Logger().Error("Unknown device type") - continue - } - + // Create Storage structure + vol, err := k.createBlkStorageObject(c, m) if vol == nil || err != nil { return nil, err } + // The device will be mounted at a unique location within the VM. Mounting + // to the container specific location is handled within the OCI spec. Let's ensure that + // the storage mount point is unique, and that this is utilized as the source in the OCI + // spec. + filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(vol.MountPoint)) + path := filepath.Join(kataGuestSandboxStorageDir(), filename) + + // Update applicable OCI mount source + for idx, ociMount := range spec.Mounts { + if ociMount.Destination != vol.MountPoint { + continue + } + k.Logger().WithFields(logrus.Fields{ + "original-source": ociMount.Source, + "new-source": path, + }).Debug("Replacing OCI mount source") + spec.Mounts[idx].Source = path + break + } + + // Update storage mountpoint, and save guest device mount path to container mount struct: + vol.MountPoint = path + c.mounts[i].GuestDeviceMount = path + volumeStorages = append(volumeStorages, vol) } diff --git a/src/runtime/virtcontainers/kata_agent_test.go b/src/runtime/virtcontainers/kata_agent_test.go index 16dd9dcfe1..9f861e932b 100644 --- a/src/runtime/virtcontainers/kata_agent_test.go +++ b/src/runtime/virtcontainers/kata_agent_test.go @@ -398,24 +398,28 @@ func TestHandleBlockVolume(t *testing.T) { containers[c.id].sandbox = &sandbox containers[c.id].mounts = mounts - volumeStorages, err := k.handleBlockVolumes(c) + vStorage, err := k.createBlkStorageObject(c, vMount) + assert.Nil(t, err, "Error while handling block volumes") + bStorage, err := k.createBlkStorageObject(c, bMount) + assert.Nil(t, err, "Error while handling block volumes") + dStorage, err := k.createBlkStorageObject(c, dMount) assert.Nil(t, err, "Error while handling block volumes") - vStorage := &pb.Storage{ + vStorageExpected := &pb.Storage{ MountPoint: vDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: vPCIPath.String(), } - bStorage := &pb.Storage{ + bStorageExpected := &pb.Storage{ MountPoint: bDestination, Fstype: "bind", Options: []string{"bind"}, Driver: kataBlkDevType, Source: bPCIPath.String(), } - dStorage := &pb.Storage{ + dStorageExpected := &pb.Storage{ MountPoint: dDestination, Fstype: "ext4", Options: []string{"ro"}, @@ -423,9 +427,9 @@ func TestHandleBlockVolume(t *testing.T) { Source: dPCIPath.String(), } - assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume") - assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume") - assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume") + assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume") + assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume") + assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume") } func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) { diff --git a/src/runtime/virtcontainers/mount.go b/src/runtime/virtcontainers/mount.go index c2879f2135..0e83bc6899 100644 --- a/src/runtime/virtcontainers/mount.go +++ b/src/runtime/virtcontainers/mount.go @@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) { }, nil } - // We get the mount point by recursively peforming stat on the path + // We get the mount point by recursively performing stat on the path // The point where the device changes indicates the mountpoint for { if mountPoint == "/" { @@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string // Mount describes a container mount. type Mount struct { - Source string + // Source is the source of the mount. + Source string + // Destination is the destination of the mount (within the container). Destination string // Type specifies the type of filesystem to mount. @@ -335,6 +337,11 @@ type Mount struct { // HostPath used to store host side bind mount path HostPath string + // GuestDeviceMount represents the path within the VM that the device + // is mounted. Only relevant for block devices. This is tracked in the event + // runtime wants to query the agent for mount stats. + GuestDeviceMount string + // BlockDeviceID represents block device that is attached to the // VM in case this mount is a block device file or a directory // backed by a block device.