runtime: update container creation to work with direct assigned volumes

During the container creation, it will parse the mount info file
of the direct assigned volumes and update the in memory mount object.

Fixes: #3454

Signed-off-by: Feng Wang <feng.wang@databricks.com>

src/runtime/virtcontainers/container.go

@@ -15,6 +15,7 @@ import (
 	"syscall"
 	"time"
 
+	volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
 	"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
 	"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
@@ -611,6 +612,28 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
 			continue
 		}
 
+		// Handle directly assigned volume. Update the mount info based on the mount info json.
+		mntInfo, e := volume.VolumeMountInfo(m.Source)
+		if e != nil && !os.IsNotExist(e) {
+			c.Logger().WithError(e).WithField("mount-source", m.Source).
+				Error("failed to parse the mount info file for a direct assigned volume")
+			continue
+		}
+
+		if mntInfo != nil {
+			// Write out sandbox info file on the mount source to allow CSI to communicate with the runtime
+			if err := volume.RecordSandboxId(c.sandboxID, m.Source); err != nil {
+				c.Logger().WithError(err).Error("error writing sandbox info")
+			}
+
+			c.mounts[i].Source = mntInfo.Device
+			c.mounts[i].Type = mntInfo.FsType
+			c.mounts[i].Options = mntInfo.Options
+			m.Source = mntInfo.Device
+			m.Type = mntInfo.FsType
+			m.Options = mntInfo.Options
+		}
+
 		var stat unix.Stat_t
 		if err := unix.Stat(m.Source, &stat); err != nil {
 			return fmt.Errorf("stat %q failed: %v", m.Source, err)

src/runtime/virtcontainers/kata_agent.go

@@ -882,35 +882,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st
 	return nil
 }
 
-func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error {
-	ociMounts := spec.Mounts
-	var index int
-	var m specs.Mount
-
-	for i, v := range volumeStorages {
-		for index, m = range ociMounts {
-			if m.Destination != v.MountPoint {
-				continue
-			}
-
-			// Create a temporary location to mount the Storage. Mounting to the correct location
-			// will be handled by the OCI mount structure.
-			filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
-			path := filepath.Join(kataGuestSandboxStorageDir(), filename)
-
-			k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
-			ociMounts[index].Source = path
-			volumeStorages[i].MountPoint = path
-
-			break
-		}
-		if index == len(ociMounts) {
-			return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint)
-		}
-	}
-	return nil
-}
-
 func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) {
 	// Disable Hooks since they have been handled on the host and there is
 	// no reason to send them to the agent. It would make no sense to try
@@ -1247,19 +1218,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
 	// Append container devices for block devices passed with --device.
 	ctrDevices = k.appendDevices(ctrDevices, c)
 
-	// Handle all the volumes that are block device files.
-	// Note this call modifies the list of container devices to make sure
-	// all hotplugged devices are unplugged, so this needs be done
-	// after devices passed with --device are handled.
-	volumeStorages, err := k.handleBlockVolumes(c)
+	// Block based volumes will require some adjustments in the OCI spec, and creation of
+	// storage objects to pass to the agent.
+	volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec)
 	if err != nil {
 		return nil, err
 	}
 
-	if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
-		return nil, err
-	}
-
 	ctrStorages = append(ctrStorages, volumeStorages...)
 
 	grpcSpec, err := grpc.OCItoGRPC(ociSpec)
@@ -1522,28 +1487,19 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D
 	vol.Options = []string{"bind"}
 	vol.MountPoint = m.Destination
 
+	// Assign the type from the mount, if it's specified (e.g. direct assigned volume)
+	if m.Type != "" {
+		vol.Fstype = m.Type
+		vol.Options = m.Options
+	}
+
 	return vol, nil
 }
 
-// handleBlockVolumes handles volumes that are block devices files
-// by passing the block devices as Storage to the agent.
-func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
-
-	var volumeStorages []*grpc.Storage
-
-	for _, m := range c.mounts {
-		id := m.BlockDeviceID
-
-		if len(id) == 0 {
-			continue
-		}
-
-		// Add the block device to the list of container devices, to make sure the
-		// device is detached with detachDevices() for a container.
-		c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
-
+func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) {
 	var vol *grpc.Storage
 
+	id := m.BlockDeviceID
 	device := c.sandbox.devManager.GetDeviceByID(id)
 	if device == nil {
 		k.Logger().WithField("device", id).Error("failed to find device by id")
@@ -1557,14 +1513,60 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
 	case config.VhostUserBlk:
 		vol, err = k.handleVhostUserBlkVolume(c, m, device)
 	default:
-			k.Logger().Error("Unknown device type")
+		return nil, fmt.Errorf("Unknown device type")
+	}
+
+	return vol, err
+}
+
+// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the
+// given container and will update the OCI spec to utilize this mount point as the new source for the
+// container volume. The container mount structure is updated to store the guest destination mountpoint.
+func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) {
+
+	var volumeStorages []*grpc.Storage
+
+	for i, m := range c.mounts {
+		id := m.BlockDeviceID
+
+		if len(id) == 0 {
 			continue
 		}
 
+		// Add the block device to the list of container devices, to make sure the
+		// device is detached with detachDevices() for a container.
+		c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
+
+		// Create Storage structure
+		vol, err := k.createBlkStorageObject(c, m)
 		if vol == nil || err != nil {
 			return nil, err
 		}
 
+		// The device will be mounted at a unique location within the VM. Mounting
+		// to the container specific location is handled within the OCI spec. Let's ensure that
+		// the storage mount point is unique, and that this is utilized as the source in the OCI
+		// spec.
+		filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(vol.MountPoint))
+		path := filepath.Join(kataGuestSandboxStorageDir(), filename)
+
+		// Update applicable OCI mount source
+		for idx, ociMount := range spec.Mounts {
+			if ociMount.Destination != vol.MountPoint {
+				continue
+			}
+			k.Logger().WithFields(logrus.Fields{
+				"original-source": ociMount.Source,
+				"new-source":      path,
+			}).Debug("Replacing OCI mount source")
+			spec.Mounts[idx].Source = path
+			break
+		}
+
+		// Update storage mountpoint, and save guest device mount path to container mount struct:
+		vol.MountPoint = path
+		c.mounts[i].GuestDeviceMount = path
+
 		volumeStorages = append(volumeStorages, vol)
 	}
 

src/runtime/virtcontainers/kata_agent_test.go

@@ -398,24 +398,28 @@ func TestHandleBlockVolume(t *testing.T) {
 	containers[c.id].sandbox = &sandbox
 	containers[c.id].mounts = mounts
 
-	volumeStorages, err := k.handleBlockVolumes(c)
+	vStorage, err := k.createBlkStorageObject(c, vMount)
+	assert.Nil(t, err, "Error while handling block volumes")
+	bStorage, err := k.createBlkStorageObject(c, bMount)
+	assert.Nil(t, err, "Error while handling block volumes")
+	dStorage, err := k.createBlkStorageObject(c, dMount)
 	assert.Nil(t, err, "Error while handling block volumes")
 
-	vStorage := &pb.Storage{
+	vStorageExpected := &pb.Storage{
 		MountPoint: vDestination,
 		Fstype:     "bind",
 		Options:    []string{"bind"},
 		Driver:     kataBlkDevType,
 		Source:     vPCIPath.String(),
 	}
-	bStorage := &pb.Storage{
+	bStorageExpected := &pb.Storage{
 		MountPoint: bDestination,
 		Fstype:     "bind",
 		Options:    []string{"bind"},
 		Driver:     kataBlkDevType,
 		Source:     bPCIPath.String(),
 	}
-	dStorage := &pb.Storage{
+	dStorageExpected := &pb.Storage{
 		MountPoint: dDestination,
 		Fstype:     "ext4",
 		Options:    []string{"ro"},
@@ -423,9 +427,9 @@ func TestHandleBlockVolume(t *testing.T) {
 		Source:     dPCIPath.String(),
 	}
 
-	assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume")
-	assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume")
-	assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume")
+	assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume")
+	assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume")
+	assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume")
 }
 
 func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) {

src/runtime/virtcontainers/mount.go

@@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) {
 		}, nil
 	}
 
-	// We get the mount point by recursively peforming stat on the path
+	// We get the mount point by recursively performing stat on the path
 	// The point where the device changes indicates the mountpoint
 	for {
 		if mountPoint == "/" {
@@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string
 
 // Mount describes a container mount.
 type Mount struct {
+	// Source is the source of the mount.
 	Source string
+	// Destination is the destination of the mount (within the container).
 	Destination string
 
 	// Type specifies the type of filesystem to mount.
@@ -335,6 +337,11 @@ type Mount struct {
 	// HostPath used to store host side bind mount path
 	HostPath string
 
+	// GuestDeviceMount represents the path within the VM that the device
+	// is mounted. Only relevant for block devices. This is tracked in the event
+	// runtime wants to query the agent for mount stats.
+	GuestDeviceMount string
+
 	// BlockDeviceID represents block device that is attached to the
 	// VM in case this mount is a block device file or a directory
 	// backed by a block device.