From eacce1d52e8f1d01e1dabb784b84748fc16b56f9 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Thu, 27 Jul 2017 14:56:57 +0100 Subject: [PATCH] Use overlay for writeable containers Previously we would sneakily remount as `rw` but of course you can't really do that on a truly immutable filesystem. See https://github.com/moby/tool/pull/129 for the `moby` side. Signed-off-by: Justin Cormack --- Makefile | 2 +- pkg/containerd/cmd/service/prepare.go | 70 ++++++++++++++++++++++++++- pkg/runc/cmd/onboot/main.go | 3 ++ pkg/runc/cmd/onboot/prepare.go | 70 ++++++++++++++++++++++++++- 4 files changed, 140 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 0babbe3d2..62669e913 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ endif PREFIX?=/usr/local/ MOBY_REPO=https://github.com/moby/tool.git -MOBY_COMMIT=36217e5145f1e54807490c09cc389f5e3ac99075 +MOBY_COMMIT=14a4d923aef7cf5a8a6f162b128315d9e0f7884e MOBY_VERSION=0.0 bin/moby: tmp_moby_bin.tar | bin tar xf $< diff --git a/pkg/containerd/cmd/service/prepare.go b/pkg/containerd/cmd/service/prepare.go index 5afde41b8..3938744c2 100644 --- a/pkg/containerd/cmd/service/prepare.go +++ b/pkg/containerd/cmd/service/prepare.go @@ -4,18 +4,84 @@ package main // Update it in both places if you make changes import ( + "fmt" + "os" "path/filepath" "syscall" ) func prepare(path string) error { + // see if we are dealing with a read only or read write container + if _, err := os.Stat(filepath.Join(path, "lower")); err != nil { + if os.IsNotExist(err) { + return prepareRO(path) + } + return err + } + return prepareRW(path) +} + +func prepareRO(path string) error { + // make rootfs a mount point, as runc doesn't like it much otherwise rootfs := filepath.Join(path, "rootfs") if err := syscall.Mount(rootfs, rootfs, "", syscall.MS_BIND, ""); err != nil { return err } - // remount rw - if err := syscall.Mount("", rootfs, "", syscall.MS_REMOUNT, ""); err != nil { + return nil +} + +func prepareRW(path string) error { + // mount a tmpfs on tmp for upper and workdirs + // make it private as nothing else should be using this + tmp := filepath.Join(path, "tmp") + if err := syscall.Mount("tmpfs", tmp, "tmpfs", 0, "size=10%"); err != nil { + return err + } + // make it private as nothing else should be using this + if err := syscall.Mount("", tmp, "", syscall.MS_REMOUNT|syscall.MS_PRIVATE, ""); err != nil { + return err + } + upper := filepath.Join(tmp, "upper") + // make the mount points + if err := os.Mkdir(upper, 0744); err != nil { + return err + } + work := filepath.Join(tmp, "work") + if err := os.Mkdir(work, 0744); err != nil { + return err + } + lower := filepath.Join(path, "lower") + rootfs := filepath.Join(path, "rootfs") + opt := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower, upper, work) + if err := syscall.Mount("overlay", rootfs, "overlay", 0, opt); err != nil { return err } return nil } + +// cleanup functions are best efforts only, mainly for rw onboot containers +func cleanup(path string) { + // see if we are dealing with a read only or read write container + if _, err := os.Stat(filepath.Join(path, "lower")); err != nil { + cleanupRO(path) + } else { + cleanupRW(path) + } +} + +func cleanupRO(path string) { + // remove the bind mount + rootfs := filepath.Join(path, "rootfs") + _ = syscall.Unmount(rootfs, 0) +} + +func cleanupRW(path string) { + // remove the overlay mount + rootfs := filepath.Join(path, "rootfs") + _ = os.RemoveAll(rootfs) + _ = syscall.Unmount(rootfs, 0) + // remove the tmpfs + tmp := filepath.Join(path, "tmp") + _ = os.RemoveAll(tmp) + _ = syscall.Unmount(tmp, 0) +} diff --git a/pkg/runc/cmd/onboot/main.go b/pkg/runc/cmd/onboot/main.go index da1a59ca7..29430be3f 100644 --- a/pkg/runc/cmd/onboot/main.go +++ b/pkg/runc/cmd/onboot/main.go @@ -56,6 +56,9 @@ func main() { if err := cmd.Run(); err != nil { log.Printf("Error running %s: %v", name, err) status = 1 + } else { + // do not clean up on error to make debug easier + cleanup(fullPath) } } diff --git a/pkg/runc/cmd/onboot/prepare.go b/pkg/runc/cmd/onboot/prepare.go index 5afde41b8..3938744c2 100644 --- a/pkg/runc/cmd/onboot/prepare.go +++ b/pkg/runc/cmd/onboot/prepare.go @@ -4,18 +4,84 @@ package main // Update it in both places if you make changes import ( + "fmt" + "os" "path/filepath" "syscall" ) func prepare(path string) error { + // see if we are dealing with a read only or read write container + if _, err := os.Stat(filepath.Join(path, "lower")); err != nil { + if os.IsNotExist(err) { + return prepareRO(path) + } + return err + } + return prepareRW(path) +} + +func prepareRO(path string) error { + // make rootfs a mount point, as runc doesn't like it much otherwise rootfs := filepath.Join(path, "rootfs") if err := syscall.Mount(rootfs, rootfs, "", syscall.MS_BIND, ""); err != nil { return err } - // remount rw - if err := syscall.Mount("", rootfs, "", syscall.MS_REMOUNT, ""); err != nil { + return nil +} + +func prepareRW(path string) error { + // mount a tmpfs on tmp for upper and workdirs + // make it private as nothing else should be using this + tmp := filepath.Join(path, "tmp") + if err := syscall.Mount("tmpfs", tmp, "tmpfs", 0, "size=10%"); err != nil { + return err + } + // make it private as nothing else should be using this + if err := syscall.Mount("", tmp, "", syscall.MS_REMOUNT|syscall.MS_PRIVATE, ""); err != nil { + return err + } + upper := filepath.Join(tmp, "upper") + // make the mount points + if err := os.Mkdir(upper, 0744); err != nil { + return err + } + work := filepath.Join(tmp, "work") + if err := os.Mkdir(work, 0744); err != nil { + return err + } + lower := filepath.Join(path, "lower") + rootfs := filepath.Join(path, "rootfs") + opt := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower, upper, work) + if err := syscall.Mount("overlay", rootfs, "overlay", 0, opt); err != nil { return err } return nil } + +// cleanup functions are best efforts only, mainly for rw onboot containers +func cleanup(path string) { + // see if we are dealing with a read only or read write container + if _, err := os.Stat(filepath.Join(path, "lower")); err != nil { + cleanupRO(path) + } else { + cleanupRW(path) + } +} + +func cleanupRO(path string) { + // remove the bind mount + rootfs := filepath.Join(path, "rootfs") + _ = syscall.Unmount(rootfs, 0) +} + +func cleanupRW(path string) { + // remove the overlay mount + rootfs := filepath.Join(path, "rootfs") + _ = os.RemoveAll(rootfs) + _ = syscall.Unmount(rootfs, 0) + // remove the tmpfs + tmp := filepath.Join(path, "tmp") + _ = os.RemoveAll(tmp) + _ = syscall.Unmount(tmp, 0) +}