Merge pull request #1412 from lifupan/shimv2mount

shimv2: optionally plug rootfs block storage instead of mounting it

cli/create.go

@@ -127,15 +127,18 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s
 
 	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
 
+	//rootfs has been mounted by containerd shim
+	rootFs := vc.RootFs{Mounted: true}
+
 	var process vc.Process
 	switch containerType {
 	case vc.PodSandbox:
-		_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
+		_, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false)
 		if err != nil {
 			return err
 		}
 	case vc.PodContainer:
-		process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, containerID, bundlePath, console, disableOutput, false)
+		process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false)
 		if err != nil {
 			return err
 		}

containerd-shim-v2/create.go

@@ -13,29 +13,34 @@ import (
 	"github.com/containerd/typeurl"
 	vc "github.com/kata-containers/runtime/virtcontainers"
 	"github.com/kata-containers/runtime/virtcontainers/pkg/oci"
+	"github.com/pkg/errors"
 	"os"
+	"path/filepath"
 
 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
 
 	"github.com/kata-containers/runtime/pkg/katautils"
 	"github.com/opencontainers/runtime-spec/specs-go"
 
+	containerd_types "github.com/containerd/containerd/api/types"
+	"github.com/containerd/containerd/mount"
+	"github.com/sirupsen/logrus"
 	// only register the proto type
 	_ "github.com/containerd/containerd/runtime/linux/runctypes"
 	crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1"
 )
 
 func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns string) (*container, error) {
-
+	rootFs := vc.RootFs{Mounted: s.mount}
-	detach := !r.Terminal
+	if len(r.Rootfs) == 1 {
-
+		m := r.Rootfs[0]
-	// Checks the MUST and MUST NOT from OCI runtime specification
+		rootFs.Source = m.Source
-	bundlePath, err := validBundle(r.ID, r.Bundle)
+		rootFs.Type = m.Type
-	if err != nil {
+		rootFs.Options = m.Options
-		return nil, err
 	}
 
-	ociSpec, err := oci.ParseConfigJSON(bundlePath)
+	detach := !r.Terminal
+	ociSpec, bundlePath, err := loadSpec(r, netns)
 	if err != nil {
 		return nil, err
 	}
@@ -45,13 +50,86 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns
 		return nil, err
 	}
 
-	// Todo:
+	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
-	// Since there is a bug in kata for sharedPidNs, here to
+	rootfs := filepath.Join(r.Bundle, "rootfs")
-	// remove the pidns to disable the sharePidNs temporarily,
+
-	// once kata fixed this issue, we can remove this line.
+	switch containerType {
-	// For the bug, please see:
+	case vc.PodSandbox:
-	// https://github.com/kata-containers/runtime/issues/930
+		if s.sandbox != nil {
-	removeNamespace(&ociSpec, specs.PIDNamespace)
+			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
+		}
+
+		_, err := loadRuntimeConfig(s, r)
+		if err != nil {
+			return nil, err
+		}
+
+		defer func() {
+			if err != nil && s.mount {
+				if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+					logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
+				}
+			}
+		}()
+
+		s.mount = true
+		if err = checkAndMount(s, r); err != nil {
+			return nil, err
+		}
+
+		rootFs.Mounted = s.mount
+
+		katautils.HandleFactory(ctx, vci, s.config)
+		sandbox, _, err := katautils.CreateSandbox(ctx, vci, *ociSpec, *s.config, rootFs, r.ID, bundlePath, "", disableOutput, false, true)
+		if err != nil {
+			return nil, err
+		}
+		s.sandbox = sandbox
+
+	case vc.PodContainer:
+		if s.sandbox == nil {
+			return nil, fmt.Errorf("BUG: Cannot start the container, since the sandbox hasn't been created")
+		}
+
+		if s.mount {
+			defer func() {
+				if err != nil {
+					if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
+						logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
+					}
+				}
+			}()
+
+			if err = doMount(r.Rootfs, rootfs); err != nil {
+				return nil, err
+			}
+		}
+
+		_, err = katautils.CreateContainer(ctx, vci, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, true)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	container, err := newContainer(s, r, containerType, ociSpec)
+	if err != nil {
+		return nil, err
+	}
+
+	return container, nil
+}
+
+func loadSpec(r *taskAPI.CreateTaskRequest, netns string) (*oci.CompatOCISpec, string, error) {
+	// Checks the MUST and MUST NOT from OCI runtime specification
+	bundlePath, err := validBundle(r.ID, r.Bundle)
+	if err != nil {
+		return nil, "", err
+	}
+
+	ociSpec, err := oci.ParseConfigJSON(bundlePath)
+	if err != nil {
+		return nil, "", err
+	}
 
 	//set the network namespace path
 	//this set will be applied to sandbox's
@@ -70,43 +148,15 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns
 		}
 	}
 
-	disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal)
+	// Todo:
+	// Since there is a bug in kata for sharedPidNs, here to
+	// remove the pidns to disable the sharePidNs temporarily,
+	// once kata fixed this issue, we can remove this line.
+	// For the bug, please see:
+	// https://github.com/kata-containers/runtime/issues/930
+	removeNamespace(&ociSpec, specs.PIDNamespace)
 
-	switch containerType {
+	return &ociSpec, bundlePath, nil
-	case vc.PodSandbox:
-		if s.sandbox != nil {
-			return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID())
-		}
-
-		_, err := loadRuntimeConfig(s, r)
-		if err != nil {
-			return nil, err
-		}
-
-		katautils.HandleFactory(ctx, vci, s.config)
-		sandbox, _, err := katautils.CreateSandbox(ctx, vci, ociSpec, *s.config, r.ID, bundlePath, "", disableOutput, false, true)
-		if err != nil {
-			return nil, err
-		}
-		s.sandbox = sandbox
-
-	case vc.PodContainer:
-		if s.sandbox == nil {
-			return nil, fmt.Errorf("BUG: Cannot start the container, since the sandbox hasn't been created")
-		}
-
-		_, err = katautils.CreateContainer(ctx, vci, s.sandbox, ociSpec, r.ID, bundlePath, "", disableOutput, true)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	container, err := newContainer(s, r, containerType, &ociSpec)
-	if err != nil {
-		return nil, err
-	}
-
-	return container, nil
 }
 
 func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeConfig, error) {
@@ -142,3 +192,33 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeCo
 
 	return &runtimeConfig, nil
 }
+
+func checkAndMount(s *service, r *taskAPI.CreateTaskRequest) error {
+	if len(r.Rootfs) == 1 {
+		m := r.Rootfs[0]
+
+		if katautils.IsBlockDevice(m.Source) && !s.config.HypervisorConfig.DisableBlockDeviceUse {
+			s.mount = false
+			return nil
+		}
+	}
+	rootfs := filepath.Join(r.Bundle, "rootfs")
+	if err := doMount(r.Rootfs, rootfs); err != nil {
+		return err
+	}
+	return nil
+}
+
+func doMount(mounts []*containerd_types.Mount, rootfs string) error {
+	for _, rm := range mounts {
+		m := &mount.Mount{
+			Type:    rm.Type,
+			Source:  rm.Source,
+			Options: rm.Options,
+		}
+		if err := m.Mount(rootfs); err != nil {
+			return errors.Wrapf(err, "failed to mount rootfs component %v", m)
+		}
+	}
+	return nil
+}

containerd-shim-v2/delete.go

@@ -39,9 +39,11 @@ func deleteContainer(ctx context.Context, s *service, c *container) error {
 		return err
 	}
 
-	rootfs := path.Join(c.bundle, "rootfs")
+	if s.mount {
-	if err := mount.UnmountAll(rootfs, 0); err != nil {
+		rootfs := path.Join(c.bundle, "rootfs")
-		logrus.WithError(err).Warn("failed to cleanup rootfs mount")
+		if err := mount.UnmountAll(rootfs, 0); err != nil {
+			logrus.WithError(err).Warn("failed to cleanup rootfs mount")
+		}
 	}
 
 	delete(s.containers, c.id)

containerd-shim-v2/service.go

@@ -10,7 +10,6 @@ import (
 	"io/ioutil"
 	"os"
 	sysexec "os/exec"
-	"path/filepath"
 	"sync"
 	"syscall"
 	"time"
@@ -18,7 +17,6 @@ import (
 	eventstypes "github.com/containerd/containerd/api/events"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/events"
-	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/namespaces"
 	cdruntime "github.com/containerd/containerd/runtime"
 	cdshim "github.com/containerd/containerd/runtime/v2/shim"
@@ -72,6 +70,7 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi
 		containers: make(map[string]*container),
 		events:     make(chan interface{}, chSize),
 		ec:         make(chan exit, bufferSize),
+		mount:      false,
 	}
 
 	go s.processExits()
@@ -99,6 +98,10 @@ type service struct {
 	// pid directly.
 	pid uint32
 
+	// if the container's rootfs is block device backed, kata shimv2
+	// will not do the rootfs mount.
+	mount bool
+
 	context    context.Context
 	sandbox    vc.VCSandbox
 	containers map[string]*container
@@ -310,39 +313,23 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
+	var c *container
+	var netns string
+
 	//the network namespace created by cni plugin
-	netns, err := namespaces.NamespaceRequired(ctx)
+	netns, err = namespaces.NamespaceRequired(ctx)
 	if err != nil {
 		return nil, errors.Wrap(err, "create namespace")
 	}
 
-	rootfs := filepath.Join(r.Bundle, "rootfs")
+	c, err = create(ctx, s, r, netns)
-	defer func() {
-		if err != nil {
-			if err2 := mount.UnmountAll(rootfs, 0); err2 != nil {
-				logrus.WithError(err2).Warn("failed to cleanup rootfs mount")
-			}
-		}
-	}()
-	for _, rm := range r.Rootfs {
-		m := &mount.Mount{
-			Type:    rm.Type,
-			Source:  rm.Source,
-			Options: rm.Options,
-		}
-		if err := m.Mount(rootfs); err != nil {
-			return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m)
-		}
-	}
-
-	container, err := create(ctx, s, r, netns)
 	if err != nil {
 		return nil, err
 	}
 
-	container.status = task.StatusCreated
+	c.status = task.StatusCreated
 
-	s.containers[r.ID] = container
+	s.containers[r.ID] = c
 
 	s.send(&eventstypes.TaskCreate{
 		ContainerID: r.ID,

pkg/katautils/create.go

@@ -169,7 +169,7 @@ func SetEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec {
 }
 
 // CreateSandbox create a sandbox container
-func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig,
+func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs,
 	containerID, bundlePath, console string, disableOutput, systemdCgroup, builtIn bool) (vc.VCSandbox, vc.Process, error) {
 	span, ctx := Trace(ctx, "createSandbox")
 	defer span.Finish()
@@ -183,6 +183,17 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, ru
 		sandboxConfig.Stateful = true
 	}
 
+	if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 {
+		if rootFs.Source != "" {
+			realPath, err := ResolvePath(rootFs.Source)
+			if err != nil {
+				return nil, vc.Process{}, err
+			}
+			rootFs.Source = realPath
+		}
+		sandboxConfig.Containers[0].RootFs = rootFs
+	}
+
 	// Important to create the network namespace before the sandbox is
 	// created, because it is not responsible for the creation of the
 	// netns if it does not exist.
@@ -223,7 +234,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, ru
 }
 
 // CreateContainer create a container
-func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) {
+func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) {
 	var c vc.VCContainer
 
 	span, ctx := Trace(ctx, "createContainer")
@@ -236,6 +247,17 @@ func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSp
 		return vc.Process{}, err
 	}
 
+	if !rootFs.Mounted {
+		if rootFs.Source != "" {
+			realPath, err := ResolvePath(rootFs.Source)
+			if err != nil {
+				return vc.Process{}, err
+			}
+			rootFs.Source = realPath
+		}
+		contConfig.RootFs = rootFs
+	}
+
 	sandboxID, err := ociSpec.SandboxID()
 	if err != nil {
 		return vc.Process{}, err

pkg/katautils/create_test.go

@@ -306,7 +306,9 @@ func TestCreateSandboxConfigFail(t *testing.T) {
 		Quota: &quota,
 	}
 
-	_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false)
+	rootFs := vc.RootFs{Mounted: true}
+
+	_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, rootFs, testContainerID, bundlePath, testConsole, true, true, false)
 	assert.Error(err)
 }
 
@@ -340,7 +342,9 @@ func TestCreateSandboxFail(t *testing.T) {
 	spec, err := readOCIConfigFile(ociConfigFile)
 	assert.NoError(err)
 
-	_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false)
+	rootFs := vc.RootFs{Mounted: true}
+
+	_, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, rootFs, testContainerID, bundlePath, testConsole, true, true, false)
 	assert.Error(err)
 	assert.True(vcmock.IsMockError(err))
 }
@@ -377,8 +381,10 @@ func TestCreateContainerContainerConfigFail(t *testing.T) {
 	err = writeOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
+	rootFs := vc.RootFs{Mounted: true}
+
 	for _, disableOutput := range []bool{true, false} {
-		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
+		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false)
 		assert.Error(err)
 		assert.False(vcmock.IsMockError(err))
 		assert.True(strings.Contains(err.Error(), containerType))
@@ -418,8 +424,10 @@ func TestCreateContainerFail(t *testing.T) {
 	err = writeOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
+	rootFs := vc.RootFs{Mounted: true}
+
 	for _, disableOutput := range []bool{true, false} {
-		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
+		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false)
 		assert.Error(err)
 		assert.True(vcmock.IsMockError(err))
 		os.RemoveAll(path)
@@ -466,8 +474,10 @@ func TestCreateContainer(t *testing.T) {
 	err = writeOCIConfigFile(spec, ociConfigFile)
 	assert.NoError(err)
 
+	rootFs := vc.RootFs{Mounted: true}
+
 	for _, disableOutput := range []bool{true, false} {
-		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false)
+		_, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false)
 		assert.NoError(err)
 		os.RemoveAll(path)
 	}

pkg/katautils/utils.go

@@ -8,6 +8,7 @@ package katautils
 
 import (
 	"fmt"
+	"golang.org/x/sys/unix"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -78,6 +79,29 @@ func ResolvePath(path string) (string, error) {
 	return resolved, nil
 }
 
+// IsBlockDevice returns true if the give path is a block device
+func IsBlockDevice(filePath string) bool {
+	var stat unix.Stat_t
+
+	if filePath == "" {
+		return false
+	}
+
+	devicePath, err := ResolvePath(filePath)
+	if err != nil {
+		return false
+	}
+
+	if err := unix.Stat(devicePath, &stat); err != nil {
+		return false
+	}
+
+	if stat.Mode&unix.S_IFBLK == unix.S_IFBLK {
+		return true
+	}
+	return false
+}
+
 // fileSize returns the number of bytes in the specified file
 func fileSize(file string) (int64, error) {
 	st := syscall.Stat_t{}

virtcontainers/api.go

@@ -589,7 +589,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err
 				State:       container.state,
 				PID:         container.process.Pid,
 				StartTime:   container.process.StartTime,
-				RootFs:      container.config.RootFs,
+				RootFs:      container.config.RootFs.Target,
 				Annotations: container.config.Annotations,
 			}, nil
 		}

virtcontainers/api_test.go

@@ -65,7 +65,7 @@ func newTestSandboxConfigNoop() SandboxConfig {
 	// Define the container command and bundle.
 	container := ContainerConfig{
 		ID:          containerID,
-		RootFs:      filepath.Join(testDir, testBundle),
+		RootFs:      RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true},
 		Cmd:         newBasicTestCmd(),
 		Annotations: containerAnnotations,
 	}
@@ -98,7 +98,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig {
 	// Define the container command and bundle.
 	container := ContainerConfig{
 		ID:          containerID,
-		RootFs:      filepath.Join(testDir, testBundle),
+		RootFs:      RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true},
 		Cmd:         newBasicTestCmd(),
 		Annotations: containerAnnotations,
 	}
@@ -136,7 +136,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig {
 	// Define the container command and bundle.
 	container := ContainerConfig{
 		ID:          containerID,
-		RootFs:      filepath.Join(testDir, testBundle),
+		RootFs:      RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true},
 		Cmd:         newBasicTestCmd(),
 		Annotations: containerAnnotations,
 	}
@@ -1026,7 +1026,7 @@ func newTestContainerConfigNoop(contID string) ContainerConfig {
 	// Define the container command and bundle.
 	container := ContainerConfig{
 		ID:          contID,
-		RootFs:      filepath.Join(testDir, testBundle),
+		RootFs:      RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true},
 		Cmd:         newBasicTestCmd(),
 		Annotations: containerAnnotations,
 	}
@@ -2053,7 +2053,7 @@ func createNewContainerConfigs(numOfContainers int) []ContainerConfig {
 		return nil
 	}
 
-	rootFs := filepath.Dir(thisFile) + "/utils/supportfiles/bundles/busybox/"
+	rootFs := RootFs{Target: filepath.Dir(thisFile) + "/utils/supportfiles/bundles/busybox/", Mounted: true}
 
 	for i := 0; i < numOfContainers; i++ {
 		contConfig := ContainerConfig{

virtcontainers/container.go

@@ -208,7 +208,7 @@ type ContainerConfig struct {
 	ID string
 
 	// RootFs is the container workload image on the host.
-	RootFs string
+	RootFs RootFs
 
 	// ReadOnlyRootfs indicates if the rootfs should be mounted readonly
 	ReadonlyRootfs bool
@@ -261,13 +261,27 @@ type ContainerDevice struct {
 	ContainerPath string
 }
 
+// RootFs describes the container's rootfs.
+type RootFs struct {
+	// Source specifies the BlockDevice path
+	Source string
+	// Target specify where the rootfs is mounted if it has been mounted
+	Target string
+	// Type specifies the type of filesystem to mount.
+	Type string
+	// Options specifies zero or more fstab style mount options.
+	Options []string
+	// Mounted specifies whether the rootfs has be mounted or not
+	Mounted bool
+}
+
 // Container is composed of a set of containers and a runtime environment.
 // A Container can be created, deleted, started, stopped, listed, entered, paused and restored.
 type Container struct {
 	id        string
 	sandboxID string
 
-	rootFs string
+	rootFs RootFs
 
 	config *ContainerConfig
 
@@ -1108,7 +1122,17 @@ func (c *Container) resume() error {
 }
 
 func (c *Container) hotplugDrive() error {
-	dev, err := getDeviceForPath(c.rootFs)
+	var dev device
+	var err error
+
+	// container rootfs is blockdevice backed and isn't mounted
+	if !c.rootFs.Mounted {
+		dev, err = getDeviceForPath(c.rootFs.Source)
+		// there is no "rootfs" dir on block device backed rootfs
+		c.rootfsSuffix = ""
+	} else {
+		dev, err = getDeviceForPath(c.rootFs.Target)
+	}
 
 	if err == errMountPointNotFound {
 		return nil
@@ -1133,14 +1157,17 @@ func (c *Container) hotplugDrive() error {
 		return nil
 	}
 
-	if dev.mountPoint == c.rootFs {
+	devicePath := c.rootFs.Source
-		c.rootfsSuffix = ""
+	fsType := c.rootFs.Type
-	}
+	if c.rootFs.Mounted {
-
+		if dev.mountPoint == c.rootFs.Target {
-	// If device mapper device, then fetch the full path of the device
+			c.rootfsSuffix = ""
-	devicePath, fsType, err := GetDevicePathAndFsType(dev.mountPoint)
+		}
-	if err != nil {
+		// If device mapper device, then fetch the full path of the device
-		return err
+		devicePath, fsType, err = GetDevicePathAndFsType(dev.mountPoint)
+		if err != nil {
+			return err
+		}
 	}
 
 	devicePath, err = filepath.EvalSymlinks(devicePath)
@@ -1153,6 +1180,14 @@ func (c *Container) hotplugDrive() error {
 		"fs-type":     fsType,
 	}).Info("Block device detected")
 
+	if err = c.plugDevice(devicePath); err != nil {
+		return err
+	}
+
+	return c.setStateFstype(fsType)
+}
+
+func (c *Container) plugDevice(devicePath string) error {
 	var stat unix.Stat_t
 	if err := unix.Stat(devicePath, &stat); err != nil {
 		return fmt.Errorf("stat %q failed: %v", devicePath, err)
@@ -1181,8 +1216,7 @@ func (c *Container) hotplugDrive() error {
 			return err
 		}
 	}
-
+	return nil
-	return c.setStateFstype(fsType)
 }
 
 // isDriveUsed checks if a drive has been used for container rootfs

virtcontainers/container_test.go

@@ -236,7 +236,7 @@ func TestContainerAddDriveDir(t *testing.T) {
 	container := Container{
 		sandbox: sandbox,
 		id:      contID,
-		rootFs:  fakeRootfs,
+		rootFs:  RootFs{Target: fakeRootfs, Mounted: true},
 	}
 
 	containerStore, err := store.NewVCContainerStore(sandbox.ctx, sandbox.id, container.id)
@@ -306,7 +306,7 @@ func TestContainerRootfsPath(t *testing.T) {
 	container := Container{
 		id:           "rootfstestcontainerid",
 		sandbox:      sandbox,
-		rootFs:       fakeRootfs,
+		rootFs:       RootFs{Target: fakeRootfs, Mounted: true},
 		rootfsSuffix: "rootfs",
 	}
 	cvcstore, err := store.NewVCContainerStore(context.Background(),
@@ -319,7 +319,7 @@ func TestContainerRootfsPath(t *testing.T) {
 	assert.Empty(t, container.rootfsSuffix)
 
 	// Reset the value to test the other case
-	container.rootFs = fakeRootfs + "/rootfs"
+	container.rootFs = RootFs{Target: fakeRootfs + "/rootfs", Mounted: true}
 	container.rootfsSuffix = "rootfs"
 
 	container.hotplugDrive()

virtcontainers/example_pod_run_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/kata-containers/runtime/virtcontainers/types"
 )
 
-const containerRootfs = "/var/lib/container/bundle/"
+var containerRootfs = vc.RootFs{Target: "/var/lib/container/bundle/", Mounted: true}
 
 // This example creates and starts a single container sandbox,
 // using qemu as the hypervisor and hyperstart as the VM agent.

virtcontainers/hack/virtc/main.go

@@ -594,7 +594,7 @@ func createContainer(context *cli.Context) error {
 
 	containerConfig := vc.ContainerConfig{
 		ID:     id,
-		RootFs: context.String("rootfs"),
+		RootFs: vc.RootFs{Target: context.String("rootfs"), Mounted: true},
 		Cmd:    cmd,
 	}
 

virtcontainers/hyperstart_agent.go

@@ -546,7 +546,7 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error {
 		container.Fstype = c.state.Fstype
 	} else {
 
-		if err := bindMountContainerRootfs(c.ctx, defaultSharedDir, sandbox.id, c.id, c.rootFs, false); err != nil {
+		if err := bindMountContainerRootfs(c.ctx, defaultSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil {
 			bindUnmountAllRootfs(c.ctx, defaultSharedDir, sandbox)
 			return err
 		}

virtcontainers/kata_agent.go

@@ -982,7 +982,7 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat
 	// (kataGuestSharedDir) is already mounted in the
 	// guest. We only need to mount the rootfs from
 	// the host and it will show up in the guest.
-	if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs, false); err != nil {
+	if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil {
 		return nil, err
 	}
 

virtcontainers/mount.go

@@ -86,10 +86,21 @@ var errMountPointNotFound = errors.New("Mount point not found")
 //
 //	device {
 //		major : major(/dev/sda1)
-//		manor : minor(/dev/sda1)
+//		minor : minor(/dev/sda1)
 //		mountPoint: /a/b/c
 //	}
+//
+//	if the path is a device path file such as /dev/sda1, it would return
+//
+//	device {
+//		major : major(/dev/sda1)
+//		minor : minor(/dev/sda1)
+//		mountPoint:
+
 func getDeviceForPath(path string) (device, error) {
+	var devMajor int
+	var devMinor int
+
 	if path == "" {
 		return device{}, fmt.Errorf("Path cannot be empty")
 	}
@@ -100,9 +111,20 @@ func getDeviceForPath(path string) (device, error) {
 		return device{}, err
 	}
 
+	if isHostDevice(path) {
+		// stat.Rdev describes the device that this file (inode) represents.
+		devMajor = major(stat.Rdev)
+		devMinor = minor(stat.Rdev)
+
+		return device{
+			major:      devMajor,
+			minor:      devMinor,
+			mountPoint: "",
+		}, nil
+	}
 	// stat.Dev points to the underlying device containing the file
-	major := major(stat.Dev)
+	devMajor = major(stat.Dev)
-	minor := minor(stat.Dev)
+	devMinor = minor(stat.Dev)
 
 	path, err = filepath.Abs(path)
 	if err != nil {
@@ -113,8 +135,8 @@ func getDeviceForPath(path string) (device, error) {
 
 	if path == "/" {
 		return device{
-			major:      major,
+			major:      devMajor,
-			minor:      minor,
+			minor:      devMinor,
 			mountPoint: mountPoint,
 		}, nil
 	}
@@ -144,8 +166,8 @@ func getDeviceForPath(path string) (device, error) {
 	}
 
 	dev := device{
-		major:      major,
+		major:      devMajor,
-		minor:      minor,
+		minor:      devMinor,
 		mountPoint: mountPoint,
 	}
 

virtcontainers/pkg/oci/utils.go

@@ -516,17 +516,17 @@ func SandboxConfig(ocispec CompatOCISpec, runtime RuntimeConfig, bundlePath, cid
 // ContainerConfig converts an OCI compatible runtime configuration
 // file to a virtcontainers container configuration structure.
 func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) {
-
 	ociSpecJSON, err := json.Marshal(ocispec)
 	if err != nil {
 		return vc.ContainerConfig{}, err
 	}
 
-	rootfs := ocispec.Root.Path
+	rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true}
-	if !filepath.IsAbs(rootfs) {
+	if !filepath.IsAbs(rootfs.Target) {
-		rootfs = filepath.Join(bundlePath, ocispec.Root.Path)
+		rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path)
 	}
-	ociLog.Debugf("container rootfs: %s", rootfs)
+
+	ociLog.Debugf("container rootfs: %s", rootfs.Target)
 
 	cmd := types.Cmd{
 		Args:            ocispec.Process.Args,

virtcontainers/pkg/oci/utils_test.go

@@ -200,7 +200,7 @@ func TestMinimalSandboxConfig(t *testing.T) {
 
 	expectedContainerConfig := vc.ContainerConfig{
 		ID:             containerID,
-		RootFs:         path.Join(tempBundlePath, "rootfs"),
+		RootFs:         vc.RootFs{Target: path.Join(tempBundlePath, "rootfs"), Mounted: true},
 		ReadonlyRootfs: true,
 		Cmd:            expectedCmd,
 		Annotations: map[string]string{

virtcontainers/sandbox.go

@@ -284,12 +284,17 @@ func (s *Sandbox) releaseStatelessSandbox() error {
 func (s *Sandbox) Status() SandboxStatus {
 	var contStatusList []ContainerStatus
 	for _, c := range s.containers {
+		rootfs := c.config.RootFs.Source
+		if c.config.RootFs.Mounted {
+			rootfs = c.config.RootFs.Target
+		}
+
 		contStatusList = append(contStatusList, ContainerStatus{
 			ID:          c.id,
 			State:       c.state,
 			PID:         c.process.Pid,
 			StartTime:   c.process.StartTime,
-			RootFs:      c.config.RootFs,
+			RootFs:      rootfs,
 			Annotations: c.config.Annotations,
 		})
 	}
@@ -1170,13 +1175,17 @@ func (s *Sandbox) StatusContainer(containerID string) (ContainerStatus, error) {
 	}
 
 	for id, c := range s.containers {
+		rootfs := c.config.RootFs.Source
+		if c.config.RootFs.Mounted {
+			rootfs = c.config.RootFs.Target
+		}
 		if id == containerID {
 			return ContainerStatus{
 				ID:          c.id,
 				State:       c.state,
 				PID:         c.process.Pid,
 				StartTime:   c.process.StartTime,
-				RootFs:      c.config.RootFs,
+				RootFs:      rootfs,
 				Annotations: c.config.Annotations,
 			}, nil
 		}