diff --git a/cli/create.go b/cli/create.go index 5f719299e1..77d356a106 100644 --- a/cli/create.go +++ b/cli/create.go @@ -127,15 +127,18 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) + //rootfs has been mounted by containerd shim + rootFs := vc.RootFs{Mounted: true} + var process vc.Process switch containerType { case vc.PodSandbox: - _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup, false) + _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false) if err != nil { return err } case vc.PodContainer: - process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, containerID, bundlePath, console, disableOutput, false) + process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false) if err != nil { return err } diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go index 5283b31239..94e51104a4 100644 --- a/pkg/katautils/create.go +++ b/pkg/katautils/create.go @@ -164,7 +164,7 @@ func SetEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec { } // CreateSandbox create a sandbox container -func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, +func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput, systemdCgroup, builtIn bool) (vc.VCSandbox, vc.Process, error) { span, ctx := Trace(ctx, "createSandbox") defer span.Finish() @@ -178,6 +178,17 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, ru sandboxConfig.Stateful = true } + if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 { + if rootFs.Source != "" { + realPath, err := ResolvePath(rootFs.Source) + if err != nil { + return nil, vc.Process{}, err + } + rootFs.Source = realPath + } + sandboxConfig.Containers[0].RootFs = rootFs + } + // Important to create the network namespace before the sandbox is // created, because it is not responsible for the creation of the // netns if it does not exist. @@ -218,7 +229,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, ru } // CreateContainer create a container -func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) { +func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) { var c vc.VCContainer span, ctx := Trace(ctx, "createContainer") @@ -231,6 +242,17 @@ func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSp return vc.Process{}, err } + if !rootFs.Mounted { + if rootFs.Source != "" { + realPath, err := ResolvePath(rootFs.Source) + if err != nil { + return vc.Process{}, err + } + rootFs.Source = realPath + } + contConfig.RootFs = rootFs + } + sandboxID, err := ociSpec.SandboxID() if err != nil { return vc.Process{}, err diff --git a/pkg/katautils/create_test.go b/pkg/katautils/create_test.go index 3caa54d3e2..364edbe1ab 100644 --- a/pkg/katautils/create_test.go +++ b/pkg/katautils/create_test.go @@ -306,7 +306,9 @@ func TestCreateSandboxConfigFail(t *testing.T) { Quota: "a, } - _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false) + rootFs := vc.RootFs{Mounted: true} + + _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, rootFs, testContainerID, bundlePath, testConsole, true, true, false) assert.Error(err) } @@ -340,7 +342,9 @@ func TestCreateSandboxFail(t *testing.T) { spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) - _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false) + rootFs := vc.RootFs{Mounted: true} + + _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, rootFs, testContainerID, bundlePath, testConsole, true, true, false) assert.Error(err) assert.True(vcmock.IsMockError(err)) } @@ -377,8 +381,10 @@ func TestCreateContainerContainerConfigFail(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) + rootFs := vc.RootFs{Mounted: true} + for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.False(vcmock.IsMockError(err)) assert.True(strings.Contains(err.Error(), containerType)) @@ -418,8 +424,10 @@ func TestCreateContainerFail(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) + rootFs := vc.RootFs{Mounted: true} + for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.True(vcmock.IsMockError(err)) os.RemoveAll(path) @@ -466,8 +474,10 @@ func TestCreateContainer(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) + rootFs := vc.RootFs{Mounted: true} + for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.NoError(err) os.RemoveAll(path) } diff --git a/pkg/katautils/utils.go b/pkg/katautils/utils.go index ae3ccecd36..e83c53da35 100644 --- a/pkg/katautils/utils.go +++ b/pkg/katautils/utils.go @@ -8,6 +8,7 @@ package katautils import ( "fmt" + "golang.org/x/sys/unix" "io/ioutil" "os" "os/exec" @@ -78,6 +79,29 @@ func ResolvePath(path string) (string, error) { return resolved, nil } +// IsBlockDevice returns true if the give path is a block device +func IsBlockDevice(filePath string) bool { + var stat unix.Stat_t + + if filePath == "" { + return false + } + + devicePath, err := ResolvePath(filePath) + if err != nil { + return false + } + + if err := unix.Stat(devicePath, &stat); err != nil { + return false + } + + if stat.Mode&unix.S_IFBLK == unix.S_IFBLK { + return true + } + return false +} + // fileSize returns the number of bytes in the specified file func fileSize(file string) (int64, error) { st := syscall.Stat_t{} diff --git a/virtcontainers/api.go b/virtcontainers/api.go index 67ce10ca94..6405a34d28 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -589,7 +589,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err State: container.state, PID: container.process.Pid, StartTime: container.process.StartTime, - RootFs: container.config.RootFs, + RootFs: container.config.RootFs.Target, Annotations: container.config.Annotations, }, nil } diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index 8e27d727cd..3893412ac3 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -65,7 +65,7 @@ func newTestSandboxConfigNoop() SandboxConfig { // Define the container command and bundle. container := ContainerConfig{ ID: containerID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -98,7 +98,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig { // Define the container command and bundle. container := ContainerConfig{ ID: containerID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -136,7 +136,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { // Define the container command and bundle. container := ContainerConfig{ ID: containerID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -1026,7 +1026,7 @@ func newTestContainerConfigNoop(contID string) ContainerConfig { // Define the container command and bundle. container := ContainerConfig{ ID: contID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -2053,7 +2053,7 @@ func createNewContainerConfigs(numOfContainers int) []ContainerConfig { return nil } - rootFs := filepath.Dir(thisFile) + "/utils/supportfiles/bundles/busybox/" + rootFs := RootFs{Target: filepath.Dir(thisFile) + "/utils/supportfiles/bundles/busybox/", Mounted: true} for i := 0; i < numOfContainers; i++ { contConfig := ContainerConfig{ diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 337d07dec7..2fe3fc1a66 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -208,7 +208,7 @@ type ContainerConfig struct { ID string // RootFs is the container workload image on the host. - RootFs string + RootFs RootFs // ReadOnlyRootfs indicates if the rootfs should be mounted readonly ReadonlyRootfs bool @@ -261,13 +261,27 @@ type ContainerDevice struct { ContainerPath string } +// RootFs describes the container's rootfs. +type RootFs struct { + // Source specifies the BlockDevice path + Source string + // Target specify where the rootfs is mounted if it has been mounted + Target string + // Type specifies the type of filesystem to mount. + Type string + // Options specifies zero or more fstab style mount options. + Options []string + // Mounted specifies whether the rootfs has be mounted or not + Mounted bool +} + // Container is composed of a set of containers and a runtime environment. // A Container can be created, deleted, started, stopped, listed, entered, paused and restored. type Container struct { id string sandboxID string - rootFs string + rootFs RootFs config *ContainerConfig @@ -1108,7 +1122,17 @@ func (c *Container) resume() error { } func (c *Container) hotplugDrive() error { - dev, err := getDeviceForPath(c.rootFs) + var dev device + var err error + + // container rootfs is blockdevice backed and isn't mounted + if !c.rootFs.Mounted { + dev, err = getDeviceForPath(c.rootFs.Source) + // there is no "rootfs" dir on block device backed rootfs + c.rootfsSuffix = "" + } else { + dev, err = getDeviceForPath(c.rootFs.Target) + } if err == errMountPointNotFound { return nil @@ -1133,14 +1157,17 @@ func (c *Container) hotplugDrive() error { return nil } - if dev.mountPoint == c.rootFs { - c.rootfsSuffix = "" - } - - // If device mapper device, then fetch the full path of the device - devicePath, fsType, err := GetDevicePathAndFsType(dev.mountPoint) - if err != nil { - return err + devicePath := c.rootFs.Source + fsType := c.rootFs.Type + if c.rootFs.Mounted { + if dev.mountPoint == c.rootFs.Target { + c.rootfsSuffix = "" + } + // If device mapper device, then fetch the full path of the device + devicePath, fsType, err = GetDevicePathAndFsType(dev.mountPoint) + if err != nil { + return err + } } devicePath, err = filepath.EvalSymlinks(devicePath) @@ -1153,6 +1180,14 @@ func (c *Container) hotplugDrive() error { "fs-type": fsType, }).Info("Block device detected") + if err = c.plugDevice(devicePath); err != nil { + return err + } + + return c.setStateFstype(fsType) +} + +func (c *Container) plugDevice(devicePath string) error { var stat unix.Stat_t if err := unix.Stat(devicePath, &stat); err != nil { return fmt.Errorf("stat %q failed: %v", devicePath, err) @@ -1181,8 +1216,7 @@ func (c *Container) hotplugDrive() error { return err } } - - return c.setStateFstype(fsType) + return nil } // isDriveUsed checks if a drive has been used for container rootfs diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index 28a56be775..8dfdafc384 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -236,7 +236,7 @@ func TestContainerAddDriveDir(t *testing.T) { container := Container{ sandbox: sandbox, id: contID, - rootFs: fakeRootfs, + rootFs: RootFs{Target: fakeRootfs, Mounted: true}, } containerStore, err := store.NewVCContainerStore(sandbox.ctx, sandbox.id, container.id) @@ -306,7 +306,7 @@ func TestContainerRootfsPath(t *testing.T) { container := Container{ id: "rootfstestcontainerid", sandbox: sandbox, - rootFs: fakeRootfs, + rootFs: RootFs{Target: fakeRootfs, Mounted: true}, rootfsSuffix: "rootfs", } cvcstore, err := store.NewVCContainerStore(context.Background(), @@ -319,7 +319,7 @@ func TestContainerRootfsPath(t *testing.T) { assert.Empty(t, container.rootfsSuffix) // Reset the value to test the other case - container.rootFs = fakeRootfs + "/rootfs" + container.rootFs = RootFs{Target: fakeRootfs + "/rootfs", Mounted: true} container.rootfsSuffix = "rootfs" container.hotplugDrive() diff --git a/virtcontainers/example_pod_run_test.go b/virtcontainers/example_pod_run_test.go index 5b93479079..f9399f6df1 100644 --- a/virtcontainers/example_pod_run_test.go +++ b/virtcontainers/example_pod_run_test.go @@ -14,7 +14,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/types" ) -const containerRootfs = "/var/lib/container/bundle/" +var containerRootfs = vc.RootFs{Target: "/var/lib/container/bundle/", Mounted: true} // This example creates and starts a single container sandbox, // using qemu as the hypervisor and hyperstart as the VM agent. diff --git a/virtcontainers/hack/virtc/main.go b/virtcontainers/hack/virtc/main.go index 2c25d43031..8329486937 100644 --- a/virtcontainers/hack/virtc/main.go +++ b/virtcontainers/hack/virtc/main.go @@ -594,7 +594,7 @@ func createContainer(context *cli.Context) error { containerConfig := vc.ContainerConfig{ ID: id, - RootFs: context.String("rootfs"), + RootFs: vc.RootFs{Target: context.String("rootfs"), Mounted: true}, Cmd: cmd, } diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index dc9f3aed40..95790c5bd9 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -546,7 +546,7 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error { container.Fstype = c.state.Fstype } else { - if err := bindMountContainerRootfs(c.ctx, defaultSharedDir, sandbox.id, c.id, c.rootFs, false); err != nil { + if err := bindMountContainerRootfs(c.ctx, defaultSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil { bindUnmountAllRootfs(c.ctx, defaultSharedDir, sandbox) return err } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 84352d84b7..b4cc4f3d10 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -982,7 +982,7 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat // (kataGuestSharedDir) is already mounted in the // guest. We only need to mount the rootfs from // the host and it will show up in the guest. - if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs, false); err != nil { + if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil { return nil, err } diff --git a/virtcontainers/mount.go b/virtcontainers/mount.go index 5ffa7c5713..a6881b059c 100644 --- a/virtcontainers/mount.go +++ b/virtcontainers/mount.go @@ -86,10 +86,21 @@ var errMountPointNotFound = errors.New("Mount point not found") // // device { // major : major(/dev/sda1) -// manor : minor(/dev/sda1) +// minor : minor(/dev/sda1) // mountPoint: /a/b/c // } +// +// if the path is a device path file such as /dev/sda1, it would return +// +// device { +// major : major(/dev/sda1) +// minor : minor(/dev/sda1) +// mountPoint: + func getDeviceForPath(path string) (device, error) { + var devMajor int + var devMinor int + if path == "" { return device{}, fmt.Errorf("Path cannot be empty") } @@ -100,9 +111,20 @@ func getDeviceForPath(path string) (device, error) { return device{}, err } + if isHostDevice(path) { + // stat.Rdev describes the device that this file (inode) represents. + devMajor = major(stat.Rdev) + devMinor = minor(stat.Rdev) + + return device{ + major: devMajor, + minor: devMinor, + mountPoint: "", + }, nil + } // stat.Dev points to the underlying device containing the file - major := major(stat.Dev) - minor := minor(stat.Dev) + devMajor = major(stat.Dev) + devMinor = minor(stat.Dev) path, err = filepath.Abs(path) if err != nil { @@ -113,8 +135,8 @@ func getDeviceForPath(path string) (device, error) { if path == "/" { return device{ - major: major, - minor: minor, + major: devMajor, + minor: devMinor, mountPoint: mountPoint, }, nil } @@ -144,8 +166,8 @@ func getDeviceForPath(path string) (device, error) { } dev := device{ - major: major, - minor: minor, + major: devMajor, + minor: devMinor, mountPoint: mountPoint, } diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 269edc9620..62e3f1e576 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -510,17 +510,17 @@ func SandboxConfig(ocispec CompatOCISpec, runtime RuntimeConfig, bundlePath, cid // ContainerConfig converts an OCI compatible runtime configuration // file to a virtcontainers container configuration structure. func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) { - ociSpecJSON, err := json.Marshal(ocispec) if err != nil { return vc.ContainerConfig{}, err } - rootfs := ocispec.Root.Path - if !filepath.IsAbs(rootfs) { - rootfs = filepath.Join(bundlePath, ocispec.Root.Path) + rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true} + if !filepath.IsAbs(rootfs.Target) { + rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path) } - ociLog.Debugf("container rootfs: %s", rootfs) + + ociLog.Debugf("container rootfs: %s", rootfs.Target) cmd := types.Cmd{ Args: ocispec.Process.Args, diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index ff1daa9b64..dd5db1a91c 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -200,7 +200,7 @@ func TestMinimalSandboxConfig(t *testing.T) { expectedContainerConfig := vc.ContainerConfig{ ID: containerID, - RootFs: path.Join(tempBundlePath, "rootfs"), + RootFs: vc.RootFs{Target: path.Join(tempBundlePath, "rootfs"), Mounted: true}, ReadonlyRootfs: true, Cmd: expectedCmd, Annotations: map[string]string{ diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 630f282f41..f0fea6dd39 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -274,12 +274,17 @@ func (s *Sandbox) releaseStatelessSandbox() error { func (s *Sandbox) Status() SandboxStatus { var contStatusList []ContainerStatus for _, c := range s.containers { + rootfs := c.config.RootFs.Source + if c.config.RootFs.Mounted { + rootfs = c.config.RootFs.Target + } + contStatusList = append(contStatusList, ContainerStatus{ ID: c.id, State: c.state, PID: c.process.Pid, StartTime: c.process.StartTime, - RootFs: c.config.RootFs, + RootFs: rootfs, Annotations: c.config.Annotations, }) } @@ -1156,13 +1161,17 @@ func (s *Sandbox) StatusContainer(containerID string) (ContainerStatus, error) { } for id, c := range s.containers { + rootfs := c.config.RootFs.Source + if c.config.RootFs.Mounted { + rootfs = c.config.RootFs.Target + } if id == containerID { return ContainerStatus{ ID: c.id, State: c.state, PID: c.process.Pid, StartTime: c.process.StartTime, - RootFs: c.config.RootFs, + RootFs: rootfs, Annotations: c.config.Annotations, }, nil }