From 628ea46c58a36208b804e4101e06bf02b847cbc0 Mon Sep 17 00:00:00 2001 From: lifupan Date: Tue, 2 Apr 2019 10:54:05 +0800 Subject: [PATCH 1/2] virtcontainers: change container's rootfs from string to mount alike struct container's rootfs is a string type, which cannot represent a block storage backed rootfs which hasn't been mounted. Change it to a mount alike struct as below: RootFs struct { // Source specify the BlockDevice path Source string // Target specify where the rootfs is mounted if it has been mounted Target string // Type specifies the type of filesystem to mount. Type string // Options specifies zero or more fstab style mount options. Options []string // Mounted specifies whether the rootfs has be mounted or not Mounted bool } If the container's rootfs has been mounted as before, then this struct can be initialized as: RootFs{Target: , Mounted: true} to be compatible with previous case. Fixes:#1158 Signed-off-by: lifupan --- cli/create.go | 7 ++- pkg/katautils/create.go | 26 ++++++++++- pkg/katautils/create_test.go | 20 ++++++--- pkg/katautils/utils.go | 24 +++++++++++ virtcontainers/api.go | 2 +- virtcontainers/api_test.go | 10 ++--- virtcontainers/container.go | 60 ++++++++++++++++++++------ virtcontainers/container_test.go | 6 +-- virtcontainers/example_pod_run_test.go | 2 +- virtcontainers/hack/virtc/main.go | 2 +- virtcontainers/hyperstart_agent.go | 2 +- virtcontainers/kata_agent.go | 2 +- virtcontainers/mount.go | 36 +++++++++++++--- virtcontainers/pkg/oci/utils.go | 10 ++--- virtcontainers/pkg/oci/utils_test.go | 2 +- virtcontainers/sandbox.go | 13 +++++- 16 files changed, 174 insertions(+), 50 deletions(-) diff --git a/cli/create.go b/cli/create.go index 5f719299e1..77d356a106 100644 --- a/cli/create.go +++ b/cli/create.go @@ -127,15 +127,18 @@ func create(ctx context.Context, containerID, bundlePath, console, pidFilePath s disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) + //rootfs has been mounted by containerd shim + rootFs := vc.RootFs{Mounted: true} + var process vc.Process switch containerType { case vc.PodSandbox: - _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, containerID, bundlePath, console, disableOutput, systemdCgroup, false) + _, process, err = katautils.CreateSandbox(ctx, vci, ociSpec, runtimeConfig, rootFs, containerID, bundlePath, console, disableOutput, systemdCgroup, false) if err != nil { return err } case vc.PodContainer: - process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, containerID, bundlePath, console, disableOutput, false) + process, err = katautils.CreateContainer(ctx, vci, nil, ociSpec, rootFs, containerID, bundlePath, console, disableOutput, false) if err != nil { return err } diff --git a/pkg/katautils/create.go b/pkg/katautils/create.go index 5283b31239..94e51104a4 100644 --- a/pkg/katautils/create.go +++ b/pkg/katautils/create.go @@ -164,7 +164,7 @@ func SetEphemeralStorageType(ociSpec oci.CompatOCISpec) oci.CompatOCISpec { } // CreateSandbox create a sandbox container -func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, +func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, runtimeConfig oci.RuntimeConfig, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput, systemdCgroup, builtIn bool) (vc.VCSandbox, vc.Process, error) { span, ctx := Trace(ctx, "createSandbox") defer span.Finish() @@ -178,6 +178,17 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, ru sandboxConfig.Stateful = true } + if !rootFs.Mounted && len(sandboxConfig.Containers) == 1 { + if rootFs.Source != "" { + realPath, err := ResolvePath(rootFs.Source) + if err != nil { + return nil, vc.Process{}, err + } + rootFs.Source = realPath + } + sandboxConfig.Containers[0].RootFs = rootFs + } + // Important to create the network namespace before the sandbox is // created, because it is not responsible for the creation of the // netns if it does not exist. @@ -218,7 +229,7 @@ func CreateSandbox(ctx context.Context, vci vc.VC, ociSpec oci.CompatOCISpec, ru } // CreateContainer create a container -func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) { +func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSpec oci.CompatOCISpec, rootFs vc.RootFs, containerID, bundlePath, console string, disableOutput, builtIn bool) (vc.Process, error) { var c vc.VCContainer span, ctx := Trace(ctx, "createContainer") @@ -231,6 +242,17 @@ func CreateContainer(ctx context.Context, vci vc.VC, sandbox vc.VCSandbox, ociSp return vc.Process{}, err } + if !rootFs.Mounted { + if rootFs.Source != "" { + realPath, err := ResolvePath(rootFs.Source) + if err != nil { + return vc.Process{}, err + } + rootFs.Source = realPath + } + contConfig.RootFs = rootFs + } + sandboxID, err := ociSpec.SandboxID() if err != nil { return vc.Process{}, err diff --git a/pkg/katautils/create_test.go b/pkg/katautils/create_test.go index 3caa54d3e2..364edbe1ab 100644 --- a/pkg/katautils/create_test.go +++ b/pkg/katautils/create_test.go @@ -306,7 +306,9 @@ func TestCreateSandboxConfigFail(t *testing.T) { Quota: "a, } - _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false) + rootFs := vc.RootFs{Mounted: true} + + _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, rootFs, testContainerID, bundlePath, testConsole, true, true, false) assert.Error(err) } @@ -340,7 +342,9 @@ func TestCreateSandboxFail(t *testing.T) { spec, err := readOCIConfigFile(ociConfigFile) assert.NoError(err) - _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, testContainerID, bundlePath, testConsole, true, true, false) + rootFs := vc.RootFs{Mounted: true} + + _, _, err = CreateSandbox(context.Background(), testingImpl, spec, runtimeConfig, rootFs, testContainerID, bundlePath, testConsole, true, true, false) assert.Error(err) assert.True(vcmock.IsMockError(err)) } @@ -377,8 +381,10 @@ func TestCreateContainerContainerConfigFail(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) + rootFs := vc.RootFs{Mounted: true} + for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.False(vcmock.IsMockError(err)) assert.True(strings.Contains(err.Error(), containerType)) @@ -418,8 +424,10 @@ func TestCreateContainerFail(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) + rootFs := vc.RootFs{Mounted: true} + for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.Error(err) assert.True(vcmock.IsMockError(err)) os.RemoveAll(path) @@ -466,8 +474,10 @@ func TestCreateContainer(t *testing.T) { err = writeOCIConfigFile(spec, ociConfigFile) assert.NoError(err) + rootFs := vc.RootFs{Mounted: true} + for _, disableOutput := range []bool{true, false} { - _, err = CreateContainer(context.Background(), testingImpl, nil, spec, testContainerID, bundlePath, testConsole, disableOutput, false) + _, err = CreateContainer(context.Background(), testingImpl, nil, spec, rootFs, testContainerID, bundlePath, testConsole, disableOutput, false) assert.NoError(err) os.RemoveAll(path) } diff --git a/pkg/katautils/utils.go b/pkg/katautils/utils.go index ae3ccecd36..e83c53da35 100644 --- a/pkg/katautils/utils.go +++ b/pkg/katautils/utils.go @@ -8,6 +8,7 @@ package katautils import ( "fmt" + "golang.org/x/sys/unix" "io/ioutil" "os" "os/exec" @@ -78,6 +79,29 @@ func ResolvePath(path string) (string, error) { return resolved, nil } +// IsBlockDevice returns true if the give path is a block device +func IsBlockDevice(filePath string) bool { + var stat unix.Stat_t + + if filePath == "" { + return false + } + + devicePath, err := ResolvePath(filePath) + if err != nil { + return false + } + + if err := unix.Stat(devicePath, &stat); err != nil { + return false + } + + if stat.Mode&unix.S_IFBLK == unix.S_IFBLK { + return true + } + return false +} + // fileSize returns the number of bytes in the specified file func fileSize(file string) (int64, error) { st := syscall.Stat_t{} diff --git a/virtcontainers/api.go b/virtcontainers/api.go index 67ce10ca94..6405a34d28 100644 --- a/virtcontainers/api.go +++ b/virtcontainers/api.go @@ -589,7 +589,7 @@ func statusContainer(sandbox *Sandbox, containerID string) (ContainerStatus, err State: container.state, PID: container.process.Pid, StartTime: container.process.StartTime, - RootFs: container.config.RootFs, + RootFs: container.config.RootFs.Target, Annotations: container.config.Annotations, }, nil } diff --git a/virtcontainers/api_test.go b/virtcontainers/api_test.go index 8e27d727cd..3893412ac3 100644 --- a/virtcontainers/api_test.go +++ b/virtcontainers/api_test.go @@ -65,7 +65,7 @@ func newTestSandboxConfigNoop() SandboxConfig { // Define the container command and bundle. container := ContainerConfig{ ID: containerID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -98,7 +98,7 @@ func newTestSandboxConfigHyperstartAgent() SandboxConfig { // Define the container command and bundle. container := ContainerConfig{ ID: containerID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -136,7 +136,7 @@ func newTestSandboxConfigHyperstartAgentDefaultNetwork() SandboxConfig { // Define the container command and bundle. container := ContainerConfig{ ID: containerID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -1026,7 +1026,7 @@ func newTestContainerConfigNoop(contID string) ContainerConfig { // Define the container command and bundle. container := ContainerConfig{ ID: contID, - RootFs: filepath.Join(testDir, testBundle), + RootFs: RootFs{Target: filepath.Join(testDir, testBundle), Mounted: true}, Cmd: newBasicTestCmd(), Annotations: containerAnnotations, } @@ -2053,7 +2053,7 @@ func createNewContainerConfigs(numOfContainers int) []ContainerConfig { return nil } - rootFs := filepath.Dir(thisFile) + "/utils/supportfiles/bundles/busybox/" + rootFs := RootFs{Target: filepath.Dir(thisFile) + "/utils/supportfiles/bundles/busybox/", Mounted: true} for i := 0; i < numOfContainers; i++ { contConfig := ContainerConfig{ diff --git a/virtcontainers/container.go b/virtcontainers/container.go index 337d07dec7..2fe3fc1a66 100644 --- a/virtcontainers/container.go +++ b/virtcontainers/container.go @@ -208,7 +208,7 @@ type ContainerConfig struct { ID string // RootFs is the container workload image on the host. - RootFs string + RootFs RootFs // ReadOnlyRootfs indicates if the rootfs should be mounted readonly ReadonlyRootfs bool @@ -261,13 +261,27 @@ type ContainerDevice struct { ContainerPath string } +// RootFs describes the container's rootfs. +type RootFs struct { + // Source specifies the BlockDevice path + Source string + // Target specify where the rootfs is mounted if it has been mounted + Target string + // Type specifies the type of filesystem to mount. + Type string + // Options specifies zero or more fstab style mount options. + Options []string + // Mounted specifies whether the rootfs has be mounted or not + Mounted bool +} + // Container is composed of a set of containers and a runtime environment. // A Container can be created, deleted, started, stopped, listed, entered, paused and restored. type Container struct { id string sandboxID string - rootFs string + rootFs RootFs config *ContainerConfig @@ -1108,7 +1122,17 @@ func (c *Container) resume() error { } func (c *Container) hotplugDrive() error { - dev, err := getDeviceForPath(c.rootFs) + var dev device + var err error + + // container rootfs is blockdevice backed and isn't mounted + if !c.rootFs.Mounted { + dev, err = getDeviceForPath(c.rootFs.Source) + // there is no "rootfs" dir on block device backed rootfs + c.rootfsSuffix = "" + } else { + dev, err = getDeviceForPath(c.rootFs.Target) + } if err == errMountPointNotFound { return nil @@ -1133,14 +1157,17 @@ func (c *Container) hotplugDrive() error { return nil } - if dev.mountPoint == c.rootFs { - c.rootfsSuffix = "" - } - - // If device mapper device, then fetch the full path of the device - devicePath, fsType, err := GetDevicePathAndFsType(dev.mountPoint) - if err != nil { - return err + devicePath := c.rootFs.Source + fsType := c.rootFs.Type + if c.rootFs.Mounted { + if dev.mountPoint == c.rootFs.Target { + c.rootfsSuffix = "" + } + // If device mapper device, then fetch the full path of the device + devicePath, fsType, err = GetDevicePathAndFsType(dev.mountPoint) + if err != nil { + return err + } } devicePath, err = filepath.EvalSymlinks(devicePath) @@ -1153,6 +1180,14 @@ func (c *Container) hotplugDrive() error { "fs-type": fsType, }).Info("Block device detected") + if err = c.plugDevice(devicePath); err != nil { + return err + } + + return c.setStateFstype(fsType) +} + +func (c *Container) plugDevice(devicePath string) error { var stat unix.Stat_t if err := unix.Stat(devicePath, &stat); err != nil { return fmt.Errorf("stat %q failed: %v", devicePath, err) @@ -1181,8 +1216,7 @@ func (c *Container) hotplugDrive() error { return err } } - - return c.setStateFstype(fsType) + return nil } // isDriveUsed checks if a drive has been used for container rootfs diff --git a/virtcontainers/container_test.go b/virtcontainers/container_test.go index 28a56be775..8dfdafc384 100644 --- a/virtcontainers/container_test.go +++ b/virtcontainers/container_test.go @@ -236,7 +236,7 @@ func TestContainerAddDriveDir(t *testing.T) { container := Container{ sandbox: sandbox, id: contID, - rootFs: fakeRootfs, + rootFs: RootFs{Target: fakeRootfs, Mounted: true}, } containerStore, err := store.NewVCContainerStore(sandbox.ctx, sandbox.id, container.id) @@ -306,7 +306,7 @@ func TestContainerRootfsPath(t *testing.T) { container := Container{ id: "rootfstestcontainerid", sandbox: sandbox, - rootFs: fakeRootfs, + rootFs: RootFs{Target: fakeRootfs, Mounted: true}, rootfsSuffix: "rootfs", } cvcstore, err := store.NewVCContainerStore(context.Background(), @@ -319,7 +319,7 @@ func TestContainerRootfsPath(t *testing.T) { assert.Empty(t, container.rootfsSuffix) // Reset the value to test the other case - container.rootFs = fakeRootfs + "/rootfs" + container.rootFs = RootFs{Target: fakeRootfs + "/rootfs", Mounted: true} container.rootfsSuffix = "rootfs" container.hotplugDrive() diff --git a/virtcontainers/example_pod_run_test.go b/virtcontainers/example_pod_run_test.go index 5b93479079..f9399f6df1 100644 --- a/virtcontainers/example_pod_run_test.go +++ b/virtcontainers/example_pod_run_test.go @@ -14,7 +14,7 @@ import ( "github.com/kata-containers/runtime/virtcontainers/types" ) -const containerRootfs = "/var/lib/container/bundle/" +var containerRootfs = vc.RootFs{Target: "/var/lib/container/bundle/", Mounted: true} // This example creates and starts a single container sandbox, // using qemu as the hypervisor and hyperstart as the VM agent. diff --git a/virtcontainers/hack/virtc/main.go b/virtcontainers/hack/virtc/main.go index 2c25d43031..8329486937 100644 --- a/virtcontainers/hack/virtc/main.go +++ b/virtcontainers/hack/virtc/main.go @@ -594,7 +594,7 @@ func createContainer(context *cli.Context) error { containerConfig := vc.ContainerConfig{ ID: id, - RootFs: context.String("rootfs"), + RootFs: vc.RootFs{Target: context.String("rootfs"), Mounted: true}, Cmd: cmd, } diff --git a/virtcontainers/hyperstart_agent.go b/virtcontainers/hyperstart_agent.go index dc9f3aed40..95790c5bd9 100644 --- a/virtcontainers/hyperstart_agent.go +++ b/virtcontainers/hyperstart_agent.go @@ -546,7 +546,7 @@ func (h *hyper) startOneContainer(sandbox *Sandbox, c *Container) error { container.Fstype = c.state.Fstype } else { - if err := bindMountContainerRootfs(c.ctx, defaultSharedDir, sandbox.id, c.id, c.rootFs, false); err != nil { + if err := bindMountContainerRootfs(c.ctx, defaultSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil { bindUnmountAllRootfs(c.ctx, defaultSharedDir, sandbox) return err } diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go index 84352d84b7..b4cc4f3d10 100644 --- a/virtcontainers/kata_agent.go +++ b/virtcontainers/kata_agent.go @@ -982,7 +982,7 @@ func (k *kataAgent) buildContainerRootfs(sandbox *Sandbox, c *Container, rootPat // (kataGuestSharedDir) is already mounted in the // guest. We only need to mount the rootfs from // the host and it will show up in the guest. - if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs, false); err != nil { + if err := bindMountContainerRootfs(k.ctx, kataHostSharedDir, sandbox.id, c.id, c.rootFs.Target, false); err != nil { return nil, err } diff --git a/virtcontainers/mount.go b/virtcontainers/mount.go index 5ffa7c5713..a6881b059c 100644 --- a/virtcontainers/mount.go +++ b/virtcontainers/mount.go @@ -86,10 +86,21 @@ var errMountPointNotFound = errors.New("Mount point not found") // // device { // major : major(/dev/sda1) -// manor : minor(/dev/sda1) +// minor : minor(/dev/sda1) // mountPoint: /a/b/c // } +// +// if the path is a device path file such as /dev/sda1, it would return +// +// device { +// major : major(/dev/sda1) +// minor : minor(/dev/sda1) +// mountPoint: + func getDeviceForPath(path string) (device, error) { + var devMajor int + var devMinor int + if path == "" { return device{}, fmt.Errorf("Path cannot be empty") } @@ -100,9 +111,20 @@ func getDeviceForPath(path string) (device, error) { return device{}, err } + if isHostDevice(path) { + // stat.Rdev describes the device that this file (inode) represents. + devMajor = major(stat.Rdev) + devMinor = minor(stat.Rdev) + + return device{ + major: devMajor, + minor: devMinor, + mountPoint: "", + }, nil + } // stat.Dev points to the underlying device containing the file - major := major(stat.Dev) - minor := minor(stat.Dev) + devMajor = major(stat.Dev) + devMinor = minor(stat.Dev) path, err = filepath.Abs(path) if err != nil { @@ -113,8 +135,8 @@ func getDeviceForPath(path string) (device, error) { if path == "/" { return device{ - major: major, - minor: minor, + major: devMajor, + minor: devMinor, mountPoint: mountPoint, }, nil } @@ -144,8 +166,8 @@ func getDeviceForPath(path string) (device, error) { } dev := device{ - major: major, - minor: minor, + major: devMajor, + minor: devMinor, mountPoint: mountPoint, } diff --git a/virtcontainers/pkg/oci/utils.go b/virtcontainers/pkg/oci/utils.go index 269edc9620..62e3f1e576 100644 --- a/virtcontainers/pkg/oci/utils.go +++ b/virtcontainers/pkg/oci/utils.go @@ -510,17 +510,17 @@ func SandboxConfig(ocispec CompatOCISpec, runtime RuntimeConfig, bundlePath, cid // ContainerConfig converts an OCI compatible runtime configuration // file to a virtcontainers container configuration structure. func ContainerConfig(ocispec CompatOCISpec, bundlePath, cid, console string, detach bool) (vc.ContainerConfig, error) { - ociSpecJSON, err := json.Marshal(ocispec) if err != nil { return vc.ContainerConfig{}, err } - rootfs := ocispec.Root.Path - if !filepath.IsAbs(rootfs) { - rootfs = filepath.Join(bundlePath, ocispec.Root.Path) + rootfs := vc.RootFs{Target: ocispec.Root.Path, Mounted: true} + if !filepath.IsAbs(rootfs.Target) { + rootfs.Target = filepath.Join(bundlePath, ocispec.Root.Path) } - ociLog.Debugf("container rootfs: %s", rootfs) + + ociLog.Debugf("container rootfs: %s", rootfs.Target) cmd := types.Cmd{ Args: ocispec.Process.Args, diff --git a/virtcontainers/pkg/oci/utils_test.go b/virtcontainers/pkg/oci/utils_test.go index ff1daa9b64..dd5db1a91c 100644 --- a/virtcontainers/pkg/oci/utils_test.go +++ b/virtcontainers/pkg/oci/utils_test.go @@ -200,7 +200,7 @@ func TestMinimalSandboxConfig(t *testing.T) { expectedContainerConfig := vc.ContainerConfig{ ID: containerID, - RootFs: path.Join(tempBundlePath, "rootfs"), + RootFs: vc.RootFs{Target: path.Join(tempBundlePath, "rootfs"), Mounted: true}, ReadonlyRootfs: true, Cmd: expectedCmd, Annotations: map[string]string{ diff --git a/virtcontainers/sandbox.go b/virtcontainers/sandbox.go index 630f282f41..f0fea6dd39 100644 --- a/virtcontainers/sandbox.go +++ b/virtcontainers/sandbox.go @@ -274,12 +274,17 @@ func (s *Sandbox) releaseStatelessSandbox() error { func (s *Sandbox) Status() SandboxStatus { var contStatusList []ContainerStatus for _, c := range s.containers { + rootfs := c.config.RootFs.Source + if c.config.RootFs.Mounted { + rootfs = c.config.RootFs.Target + } + contStatusList = append(contStatusList, ContainerStatus{ ID: c.id, State: c.state, PID: c.process.Pid, StartTime: c.process.StartTime, - RootFs: c.config.RootFs, + RootFs: rootfs, Annotations: c.config.Annotations, }) } @@ -1156,13 +1161,17 @@ func (s *Sandbox) StatusContainer(containerID string) (ContainerStatus, error) { } for id, c := range s.containers { + rootfs := c.config.RootFs.Source + if c.config.RootFs.Mounted { + rootfs = c.config.RootFs.Target + } if id == containerID { return ContainerStatus{ ID: c.id, State: c.state, PID: c.process.Pid, StartTime: c.process.StartTime, - RootFs: c.config.RootFs, + RootFs: rootfs, Annotations: c.config.Annotations, }, nil } From 52c66d20dc353fdf5d367276dbe4b2d325156c15 Mon Sep 17 00:00:00 2001 From: lifupan Date: Tue, 2 Apr 2019 10:55:20 +0800 Subject: [PATCH 2/2] shimv2: plugin the block backed rootfs directly instead of mount it When the container's rootfs is block storage backed such as devmapper, shimv2 will not mount it on the host, instead it insert it into hypervisor as a block device directly. If kata's config set "disable_block_device_use" as true, it will mount the rootfs onto host as before. Fixes:#1158 Signed-off-by: lifupan --- containerd-shim-v2/create.go | 182 ++++++++++++++++++++++++---------- containerd-shim-v2/delete.go | 8 +- containerd-shim-v2/service.go | 37 +++---- 3 files changed, 148 insertions(+), 79 deletions(-) diff --git a/containerd-shim-v2/create.go b/containerd-shim-v2/create.go index 65e301135a..b5fe278b42 100644 --- a/containerd-shim-v2/create.go +++ b/containerd-shim-v2/create.go @@ -13,29 +13,34 @@ import ( "github.com/containerd/typeurl" vc "github.com/kata-containers/runtime/virtcontainers" "github.com/kata-containers/runtime/virtcontainers/pkg/oci" + "github.com/pkg/errors" "os" + "path/filepath" taskAPI "github.com/containerd/containerd/runtime/v2/task" "github.com/kata-containers/runtime/pkg/katautils" "github.com/opencontainers/runtime-spec/specs-go" + containerd_types "github.com/containerd/containerd/api/types" + "github.com/containerd/containerd/mount" + "github.com/sirupsen/logrus" // only register the proto type _ "github.com/containerd/containerd/runtime/linux/runctypes" crioption "github.com/containerd/cri-containerd/pkg/api/runtimeoptions/v1" ) func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns string) (*container, error) { - - detach := !r.Terminal - - // Checks the MUST and MUST NOT from OCI runtime specification - bundlePath, err := validBundle(r.ID, r.Bundle) - if err != nil { - return nil, err + rootFs := vc.RootFs{Mounted: s.mount} + if len(r.Rootfs) == 1 { + m := r.Rootfs[0] + rootFs.Source = m.Source + rootFs.Type = m.Type + rootFs.Options = m.Options } - ociSpec, err := oci.ParseConfigJSON(bundlePath) + detach := !r.Terminal + ociSpec, bundlePath, err := loadSpec(r, netns) if err != nil { return nil, err } @@ -45,13 +50,86 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns return nil, err } - // Todo: - // Since there is a bug in kata for sharedPidNs, here to - // remove the pidns to disable the sharePidNs temporarily, - // once kata fixed this issue, we can remove this line. - // For the bug, please see: - // https://github.com/kata-containers/runtime/issues/930 - removeNamespace(&ociSpec, specs.PIDNamespace) + disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) + rootfs := filepath.Join(r.Bundle, "rootfs") + + switch containerType { + case vc.PodSandbox: + if s.sandbox != nil { + return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID()) + } + + _, err := loadRuntimeConfig(s, r) + if err != nil { + return nil, err + } + + defer func() { + if err != nil && s.mount { + if err2 := mount.UnmountAll(rootfs, 0); err2 != nil { + logrus.WithError(err2).Warn("failed to cleanup rootfs mount") + } + } + }() + + s.mount = true + if err = checkAndMount(s, r); err != nil { + return nil, err + } + + rootFs.Mounted = s.mount + + katautils.HandleFactory(ctx, vci, s.config) + sandbox, _, err := katautils.CreateSandbox(ctx, vci, *ociSpec, *s.config, rootFs, r.ID, bundlePath, "", disableOutput, false, true) + if err != nil { + return nil, err + } + s.sandbox = sandbox + + case vc.PodContainer: + if s.sandbox == nil { + return nil, fmt.Errorf("BUG: Cannot start the container, since the sandbox hasn't been created") + } + + if s.mount { + defer func() { + if err != nil { + if err2 := mount.UnmountAll(rootfs, 0); err2 != nil { + logrus.WithError(err2).Warn("failed to cleanup rootfs mount") + } + } + }() + + if err = doMount(r.Rootfs, rootfs); err != nil { + return nil, err + } + } + + _, err = katautils.CreateContainer(ctx, vci, s.sandbox, *ociSpec, rootFs, r.ID, bundlePath, "", disableOutput, true) + if err != nil { + return nil, err + } + } + + container, err := newContainer(s, r, containerType, ociSpec) + if err != nil { + return nil, err + } + + return container, nil +} + +func loadSpec(r *taskAPI.CreateTaskRequest, netns string) (*oci.CompatOCISpec, string, error) { + // Checks the MUST and MUST NOT from OCI runtime specification + bundlePath, err := validBundle(r.ID, r.Bundle) + if err != nil { + return nil, "", err + } + + ociSpec, err := oci.ParseConfigJSON(bundlePath) + if err != nil { + return nil, "", err + } //set the network namespace path //this set will be applied to sandbox's @@ -70,43 +148,15 @@ func create(ctx context.Context, s *service, r *taskAPI.CreateTaskRequest, netns } } - disableOutput := noNeedForOutput(detach, ociSpec.Process.Terminal) + // Todo: + // Since there is a bug in kata for sharedPidNs, here to + // remove the pidns to disable the sharePidNs temporarily, + // once kata fixed this issue, we can remove this line. + // For the bug, please see: + // https://github.com/kata-containers/runtime/issues/930 + removeNamespace(&ociSpec, specs.PIDNamespace) - switch containerType { - case vc.PodSandbox: - if s.sandbox != nil { - return nil, fmt.Errorf("cannot create another sandbox in sandbox: %s", s.sandbox.ID()) - } - - _, err := loadRuntimeConfig(s, r) - if err != nil { - return nil, err - } - - katautils.HandleFactory(ctx, vci, s.config) - sandbox, _, err := katautils.CreateSandbox(ctx, vci, ociSpec, *s.config, r.ID, bundlePath, "", disableOutput, false, true) - if err != nil { - return nil, err - } - s.sandbox = sandbox - - case vc.PodContainer: - if s.sandbox == nil { - return nil, fmt.Errorf("BUG: Cannot start the container, since the sandbox hasn't been created") - } - - _, err = katautils.CreateContainer(ctx, vci, s.sandbox, ociSpec, r.ID, bundlePath, "", disableOutput, true) - if err != nil { - return nil, err - } - } - - container, err := newContainer(s, r, containerType, &ociSpec) - if err != nil { - return nil, err - } - - return container, nil + return &ociSpec, bundlePath, nil } func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeConfig, error) { @@ -142,3 +192,33 @@ func loadRuntimeConfig(s *service, r *taskAPI.CreateTaskRequest) (*oci.RuntimeCo return &runtimeConfig, nil } + +func checkAndMount(s *service, r *taskAPI.CreateTaskRequest) error { + if len(r.Rootfs) == 1 { + m := r.Rootfs[0] + + if katautils.IsBlockDevice(m.Source) && !s.config.HypervisorConfig.DisableBlockDeviceUse { + s.mount = false + return nil + } + } + rootfs := filepath.Join(r.Bundle, "rootfs") + if err := doMount(r.Rootfs, rootfs); err != nil { + return err + } + return nil +} + +func doMount(mounts []*containerd_types.Mount, rootfs string) error { + for _, rm := range mounts { + m := &mount.Mount{ + Type: rm.Type, + Source: rm.Source, + Options: rm.Options, + } + if err := m.Mount(rootfs); err != nil { + return errors.Wrapf(err, "failed to mount rootfs component %v", m) + } + } + return nil +} diff --git a/containerd-shim-v2/delete.go b/containerd-shim-v2/delete.go index 936133b5c9..462749215e 100644 --- a/containerd-shim-v2/delete.go +++ b/containerd-shim-v2/delete.go @@ -39,9 +39,11 @@ func deleteContainer(ctx context.Context, s *service, c *container) error { return err } - rootfs := path.Join(c.bundle, "rootfs") - if err := mount.UnmountAll(rootfs, 0); err != nil { - logrus.WithError(err).Warn("failed to cleanup rootfs mount") + if s.mount { + rootfs := path.Join(c.bundle, "rootfs") + if err := mount.UnmountAll(rootfs, 0); err != nil { + logrus.WithError(err).Warn("failed to cleanup rootfs mount") + } } delete(s.containers, c.id) diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go index 7ed3971ae3..e00a6d135f 100644 --- a/containerd-shim-v2/service.go +++ b/containerd-shim-v2/service.go @@ -10,7 +10,6 @@ import ( "io/ioutil" "os" sysexec "os/exec" - "path/filepath" "sync" "syscall" "time" @@ -18,7 +17,6 @@ import ( eventstypes "github.com/containerd/containerd/api/events" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/events" - "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" cdruntime "github.com/containerd/containerd/runtime" cdshim "github.com/containerd/containerd/runtime/v2/shim" @@ -72,6 +70,7 @@ func New(ctx context.Context, id string, publisher events.Publisher) (cdshim.Shi containers: make(map[string]*container), events: make(chan interface{}, chSize), ec: make(chan exit, bufferSize), + mount: false, } go s.processExits() @@ -99,6 +98,10 @@ type service struct { // pid directly. pid uint32 + // if the container's rootfs is block device backed, kata shimv2 + // will not do the rootfs mount. + mount bool + context context.Context sandbox vc.VCSandbox containers map[string]*container @@ -310,39 +313,23 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ * s.mu.Lock() defer s.mu.Unlock() + var c *container + var netns string + //the network namespace created by cni plugin - netns, err := namespaces.NamespaceRequired(ctx) + netns, err = namespaces.NamespaceRequired(ctx) if err != nil { return nil, errors.Wrap(err, "create namespace") } - rootfs := filepath.Join(r.Bundle, "rootfs") - defer func() { - if err != nil { - if err2 := mount.UnmountAll(rootfs, 0); err2 != nil { - logrus.WithError(err2).Warn("failed to cleanup rootfs mount") - } - } - }() - for _, rm := range r.Rootfs { - m := &mount.Mount{ - Type: rm.Type, - Source: rm.Source, - Options: rm.Options, - } - if err := m.Mount(rootfs); err != nil { - return nil, errors.Wrapf(err, "failed to mount rootfs component %v", m) - } - } - - container, err := create(ctx, s, r, netns) + c, err = create(ctx, s, r, netns) if err != nil { return nil, err } - container.status = task.StatusCreated + c.status = task.StatusCreated - s.containers[r.ID] = container + s.containers[r.ID] = c s.send(&eventstypes.TaskCreate{ ContainerID: r.ID,