mirror of
https://github.com/kata-containers/kata-containers.git
synced 2025-05-01 05:04:26 +00:00
runtime: update container creation to work with direct assigned volumes
During the container creation, it will parse the mount info file of the direct assigned volumes and update the in memory mount object. Fixes: #3454 Signed-off-by: Feng Wang <feng.wang@databricks.com>
This commit is contained in:
parent
4e00c2377c
commit
c39281ad65
@ -15,6 +15,7 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
volume "github.com/kata-containers/kata-containers/src/runtime/pkg/direct-volume"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/pkg/katautils/katatrace"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/config"
|
||||
"github.com/kata-containers/kata-containers/src/runtime/virtcontainers/device/manager"
|
||||
@ -611,6 +612,28 @@ func (c *Container) createBlockDevices(ctx context.Context) error {
|
||||
continue
|
||||
}
|
||||
|
||||
// Handle directly assigned volume. Update the mount info based on the mount info json.
|
||||
mntInfo, e := volume.VolumeMountInfo(m.Source)
|
||||
if e != nil && !os.IsNotExist(e) {
|
||||
c.Logger().WithError(e).WithField("mount-source", m.Source).
|
||||
Error("failed to parse the mount info file for a direct assigned volume")
|
||||
continue
|
||||
}
|
||||
|
||||
if mntInfo != nil {
|
||||
// Write out sandbox info file on the mount source to allow CSI to communicate with the runtime
|
||||
if err := volume.RecordSandboxId(c.sandboxID, m.Source); err != nil {
|
||||
c.Logger().WithError(err).Error("error writing sandbox info")
|
||||
}
|
||||
|
||||
c.mounts[i].Source = mntInfo.Device
|
||||
c.mounts[i].Type = mntInfo.FsType
|
||||
c.mounts[i].Options = mntInfo.Options
|
||||
m.Source = mntInfo.Device
|
||||
m.Type = mntInfo.FsType
|
||||
m.Options = mntInfo.Options
|
||||
}
|
||||
|
||||
var stat unix.Stat_t
|
||||
if err := unix.Stat(m.Source, &stat); err != nil {
|
||||
return fmt.Errorf("stat %q failed: %v", m.Source, err)
|
||||
|
@ -882,35 +882,6 @@ func (k *kataAgent) removeIgnoredOCIMount(spec *specs.Spec, ignoredMounts map[st
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *kataAgent) replaceOCIMountsForStorages(spec *specs.Spec, volumeStorages []*grpc.Storage) error {
|
||||
ociMounts := spec.Mounts
|
||||
var index int
|
||||
var m specs.Mount
|
||||
|
||||
for i, v := range volumeStorages {
|
||||
for index, m = range ociMounts {
|
||||
if m.Destination != v.MountPoint {
|
||||
continue
|
||||
}
|
||||
|
||||
// Create a temporary location to mount the Storage. Mounting to the correct location
|
||||
// will be handled by the OCI mount structure.
|
||||
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(m.Destination))
|
||||
path := filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||
|
||||
k.Logger().Debugf("Replacing OCI mount source (%s) with %s", m.Source, path)
|
||||
ociMounts[index].Source = path
|
||||
volumeStorages[i].MountPoint = path
|
||||
|
||||
break
|
||||
}
|
||||
if index == len(ociMounts) {
|
||||
return fmt.Errorf("OCI mount not found for block volume %s", v.MountPoint)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *kataAgent) constrainGRPCSpec(grpcSpec *grpc.Spec, passSeccomp bool, stripVfio bool) {
|
||||
// Disable Hooks since they have been handled on the host and there is
|
||||
// no reason to send them to the agent. It would make no sense to try
|
||||
@ -1247,19 +1218,13 @@ func (k *kataAgent) createContainer(ctx context.Context, sandbox *Sandbox, c *Co
|
||||
// Append container devices for block devices passed with --device.
|
||||
ctrDevices = k.appendDevices(ctrDevices, c)
|
||||
|
||||
// Handle all the volumes that are block device files.
|
||||
// Note this call modifies the list of container devices to make sure
|
||||
// all hotplugged devices are unplugged, so this needs be done
|
||||
// after devices passed with --device are handled.
|
||||
volumeStorages, err := k.handleBlockVolumes(c)
|
||||
// Block based volumes will require some adjustments in the OCI spec, and creation of
|
||||
// storage objects to pass to the agent.
|
||||
volumeStorages, err := k.handleBlkOCIMounts(c, ociSpec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := k.replaceOCIMountsForStorages(ociSpec, volumeStorages); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ctrStorages = append(ctrStorages, volumeStorages...)
|
||||
|
||||
grpcSpec, err := grpc.OCItoGRPC(ociSpec)
|
||||
@ -1522,16 +1487,46 @@ func (k *kataAgent) handleVhostUserBlkVolume(c *Container, m Mount, device api.D
|
||||
vol.Options = []string{"bind"}
|
||||
vol.MountPoint = m.Destination
|
||||
|
||||
// Assign the type from the mount, if it's specified (e.g. direct assigned volume)
|
||||
if m.Type != "" {
|
||||
vol.Fstype = m.Type
|
||||
vol.Options = m.Options
|
||||
}
|
||||
|
||||
return vol, nil
|
||||
}
|
||||
|
||||
// handleBlockVolumes handles volumes that are block devices files
|
||||
// by passing the block devices as Storage to the agent.
|
||||
func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||
func (k *kataAgent) createBlkStorageObject(c *Container, m Mount) (*grpc.Storage, error) {
|
||||
var vol *grpc.Storage
|
||||
|
||||
id := m.BlockDeviceID
|
||||
device := c.sandbox.devManager.GetDeviceByID(id)
|
||||
if device == nil {
|
||||
k.Logger().WithField("device", id).Error("failed to find device by id")
|
||||
return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
|
||||
}
|
||||
|
||||
var err error
|
||||
switch device.DeviceType() {
|
||||
case config.DeviceBlock:
|
||||
vol, err = k.handleDeviceBlockVolume(c, m, device)
|
||||
case config.VhostUserBlk:
|
||||
vol, err = k.handleVhostUserBlkVolume(c, m, device)
|
||||
default:
|
||||
return nil, fmt.Errorf("Unknown device type")
|
||||
}
|
||||
|
||||
return vol, err
|
||||
}
|
||||
|
||||
// handleBlkOCIMounts will create a unique destination mountpoint in the guest for each volume in the
|
||||
// given container and will update the OCI spec to utilize this mount point as the new source for the
|
||||
// container volume. The container mount structure is updated to store the guest destination mountpoint.
|
||||
func (k *kataAgent) handleBlkOCIMounts(c *Container, spec *specs.Spec) ([]*grpc.Storage, error) {
|
||||
|
||||
var volumeStorages []*grpc.Storage
|
||||
|
||||
for _, m := range c.mounts {
|
||||
for i, m := range c.mounts {
|
||||
id := m.BlockDeviceID
|
||||
|
||||
if len(id) == 0 {
|
||||
@ -1542,29 +1537,36 @@ func (k *kataAgent) handleBlockVolumes(c *Container) ([]*grpc.Storage, error) {
|
||||
// device is detached with detachDevices() for a container.
|
||||
c.devices = append(c.devices, ContainerDevice{ID: id, ContainerPath: m.Destination})
|
||||
|
||||
var vol *grpc.Storage
|
||||
|
||||
device := c.sandbox.devManager.GetDeviceByID(id)
|
||||
if device == nil {
|
||||
k.Logger().WithField("device", id).Error("failed to find device by id")
|
||||
return nil, fmt.Errorf("Failed to find device by id (id=%s)", id)
|
||||
}
|
||||
|
||||
var err error
|
||||
switch device.DeviceType() {
|
||||
case config.DeviceBlock:
|
||||
vol, err = k.handleDeviceBlockVolume(c, m, device)
|
||||
case config.VhostUserBlk:
|
||||
vol, err = k.handleVhostUserBlkVolume(c, m, device)
|
||||
default:
|
||||
k.Logger().Error("Unknown device type")
|
||||
continue
|
||||
}
|
||||
|
||||
// Create Storage structure
|
||||
vol, err := k.createBlkStorageObject(c, m)
|
||||
if vol == nil || err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The device will be mounted at a unique location within the VM. Mounting
|
||||
// to the container specific location is handled within the OCI spec. Let's ensure that
|
||||
// the storage mount point is unique, and that this is utilized as the source in the OCI
|
||||
// spec.
|
||||
filename := fmt.Sprintf("%s-%s", uuid.Generate().String(), filepath.Base(vol.MountPoint))
|
||||
path := filepath.Join(kataGuestSandboxStorageDir(), filename)
|
||||
|
||||
// Update applicable OCI mount source
|
||||
for idx, ociMount := range spec.Mounts {
|
||||
if ociMount.Destination != vol.MountPoint {
|
||||
continue
|
||||
}
|
||||
k.Logger().WithFields(logrus.Fields{
|
||||
"original-source": ociMount.Source,
|
||||
"new-source": path,
|
||||
}).Debug("Replacing OCI mount source")
|
||||
spec.Mounts[idx].Source = path
|
||||
break
|
||||
}
|
||||
|
||||
// Update storage mountpoint, and save guest device mount path to container mount struct:
|
||||
vol.MountPoint = path
|
||||
c.mounts[i].GuestDeviceMount = path
|
||||
|
||||
volumeStorages = append(volumeStorages, vol)
|
||||
}
|
||||
|
||||
|
@ -398,24 +398,28 @@ func TestHandleBlockVolume(t *testing.T) {
|
||||
containers[c.id].sandbox = &sandbox
|
||||
containers[c.id].mounts = mounts
|
||||
|
||||
volumeStorages, err := k.handleBlockVolumes(c)
|
||||
vStorage, err := k.createBlkStorageObject(c, vMount)
|
||||
assert.Nil(t, err, "Error while handling block volumes")
|
||||
bStorage, err := k.createBlkStorageObject(c, bMount)
|
||||
assert.Nil(t, err, "Error while handling block volumes")
|
||||
dStorage, err := k.createBlkStorageObject(c, dMount)
|
||||
assert.Nil(t, err, "Error while handling block volumes")
|
||||
|
||||
vStorage := &pb.Storage{
|
||||
vStorageExpected := &pb.Storage{
|
||||
MountPoint: vDestination,
|
||||
Fstype: "bind",
|
||||
Options: []string{"bind"},
|
||||
Driver: kataBlkDevType,
|
||||
Source: vPCIPath.String(),
|
||||
}
|
||||
bStorage := &pb.Storage{
|
||||
bStorageExpected := &pb.Storage{
|
||||
MountPoint: bDestination,
|
||||
Fstype: "bind",
|
||||
Options: []string{"bind"},
|
||||
Driver: kataBlkDevType,
|
||||
Source: bPCIPath.String(),
|
||||
}
|
||||
dStorage := &pb.Storage{
|
||||
dStorageExpected := &pb.Storage{
|
||||
MountPoint: dDestination,
|
||||
Fstype: "ext4",
|
||||
Options: []string{"ro"},
|
||||
@ -423,9 +427,9 @@ func TestHandleBlockVolume(t *testing.T) {
|
||||
Source: dPCIPath.String(),
|
||||
}
|
||||
|
||||
assert.Equal(t, vStorage, volumeStorages[0], "Error while handle VhostUserBlk type block volume")
|
||||
assert.Equal(t, bStorage, volumeStorages[1], "Error while handle BlockDevice type block volume")
|
||||
assert.Equal(t, dStorage, volumeStorages[2], "Error while handle direct BlockDevice type block volume")
|
||||
assert.Equal(t, vStorage, vStorageExpected, "Error while handle VhostUserBlk type block volume")
|
||||
assert.Equal(t, bStorage, bStorageExpected, "Error while handle BlockDevice type block volume")
|
||||
assert.Equal(t, dStorage, dStorageExpected, "Error while handle direct BlockDevice type block volume")
|
||||
}
|
||||
|
||||
func TestAppendDevicesEmptyContainerDeviceList(t *testing.T) {
|
||||
|
@ -172,7 +172,7 @@ func getDeviceForPath(path string) (device, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// We get the mount point by recursively peforming stat on the path
|
||||
// We get the mount point by recursively performing stat on the path
|
||||
// The point where the device changes indicates the mountpoint
|
||||
for {
|
||||
if mountPoint == "/" {
|
||||
@ -326,7 +326,9 @@ func bindMountContainerRootfs(ctx context.Context, shareDir, cid, cRootFs string
|
||||
|
||||
// Mount describes a container mount.
|
||||
type Mount struct {
|
||||
Source string
|
||||
// Source is the source of the mount.
|
||||
Source string
|
||||
// Destination is the destination of the mount (within the container).
|
||||
Destination string
|
||||
|
||||
// Type specifies the type of filesystem to mount.
|
||||
@ -335,6 +337,11 @@ type Mount struct {
|
||||
// HostPath used to store host side bind mount path
|
||||
HostPath string
|
||||
|
||||
// GuestDeviceMount represents the path within the VM that the device
|
||||
// is mounted. Only relevant for block devices. This is tracked in the event
|
||||
// runtime wants to query the agent for mount stats.
|
||||
GuestDeviceMount string
|
||||
|
||||
// BlockDeviceID represents block device that is attached to the
|
||||
// VM in case this mount is a block device file or a directory
|
||||
// backed by a block device.
|
||||
|
Loading…
Reference in New Issue
Block a user