Use overlay for writeable containers

Previously we would sneakily remount as `rw` but of course you can't
really do that on a truly immutable filesystem.

See https://github.com/moby/tool/pull/129 for the `moby` side.

Signed-off-by: Justin Cormack <justin.cormack@docker.com>

Makefile

@@ -21,7 +21,7 @@ endif
 PREFIX?=/usr/local/
 
 MOBY_REPO=https://github.com/moby/tool.git
-MOBY_COMMIT=36217e5145f1e54807490c09cc389f5e3ac99075
+MOBY_COMMIT=14a4d923aef7cf5a8a6f162b128315d9e0f7884e
 MOBY_VERSION=0.0
 bin/moby: tmp_moby_bin.tar | bin
 	tar xf $<

pkg/containerd/cmd/service/prepare.go

@@ -4,18 +4,84 @@ package main
 // Update it in both places if you make changes
 
 import (
+	"fmt"
+	"os"
 	"path/filepath"
 	"syscall"
 )
 
 func prepare(path string) error {
+	// see if we are dealing with a read only or read write container
+	if _, err := os.Stat(filepath.Join(path, "lower")); err != nil {
+		if os.IsNotExist(err) {
+			return prepareRO(path)
+		}
+		return err
+	}
+	return prepareRW(path)
+}
+
+func prepareRO(path string) error {
+	// make rootfs a mount point, as runc doesn't like it much otherwise
 	rootfs := filepath.Join(path, "rootfs")
 	if err := syscall.Mount(rootfs, rootfs, "", syscall.MS_BIND, ""); err != nil {
 		return err
 	}
-	// remount rw
-	if err := syscall.Mount("", rootfs, "", syscall.MS_REMOUNT, ""); err != nil {
+	return nil
+}
+
+func prepareRW(path string) error {
+	// mount a tmpfs on tmp for upper and workdirs
+	// make it private as nothing else should be using this
+	tmp := filepath.Join(path, "tmp")
+	if err := syscall.Mount("tmpfs", tmp, "tmpfs", 0, "size=10%"); err != nil {
+		return err
+	}
+	// make it private as nothing else should be using this
+	if err := syscall.Mount("", tmp, "", syscall.MS_REMOUNT|syscall.MS_PRIVATE, ""); err != nil {
+		return err
+	}
+	upper := filepath.Join(tmp, "upper")
+	// make the mount points
+	if err := os.Mkdir(upper, 0744); err != nil {
+		return err
+	}
+	work := filepath.Join(tmp, "work")
+	if err := os.Mkdir(work, 0744); err != nil {
+		return err
+	}
+	lower := filepath.Join(path, "lower")
+	rootfs := filepath.Join(path, "rootfs")
+	opt := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower, upper, work)
+	if err := syscall.Mount("overlay", rootfs, "overlay", 0, opt); err != nil {
 		return err
 	}
 	return nil
 }
+
+// cleanup functions are best efforts only, mainly for rw onboot containers
+func cleanup(path string) {
+	// see if we are dealing with a read only or read write container
+	if _, err := os.Stat(filepath.Join(path, "lower")); err != nil {
+		cleanupRO(path)
+	} else {
+		cleanupRW(path)
+	}
+}
+
+func cleanupRO(path string) {
+	// remove the bind mount
+	rootfs := filepath.Join(path, "rootfs")
+	_ = syscall.Unmount(rootfs, 0)
+}
+
+func cleanupRW(path string) {
+	// remove the overlay mount
+	rootfs := filepath.Join(path, "rootfs")
+	_ = os.RemoveAll(rootfs)
+	_ = syscall.Unmount(rootfs, 0)
+	// remove the tmpfs
+	tmp := filepath.Join(path, "tmp")
+	_ = os.RemoveAll(tmp)
+	_ = syscall.Unmount(tmp, 0)
+}

pkg/runc/cmd/onboot/main.go

@@ -56,6 +56,9 @@ func main() {
 		if err := cmd.Run(); err != nil {
 			log.Printf("Error running %s: %v", name, err)
 			status = 1
+		} else {
+			// do not clean up on error to make debug easier
+			cleanup(fullPath)
 		}
 	}
 

pkg/runc/cmd/onboot/prepare.go

@@ -4,18 +4,84 @@ package main
 // Update it in both places if you make changes
 
 import (
+	"fmt"
+	"os"
 	"path/filepath"
 	"syscall"
 )
 
 func prepare(path string) error {
+	// see if we are dealing with a read only or read write container
+	if _, err := os.Stat(filepath.Join(path, "lower")); err != nil {
+		if os.IsNotExist(err) {
+			return prepareRO(path)
+		}
+		return err
+	}
+	return prepareRW(path)
+}
+
+func prepareRO(path string) error {
+	// make rootfs a mount point, as runc doesn't like it much otherwise
 	rootfs := filepath.Join(path, "rootfs")
 	if err := syscall.Mount(rootfs, rootfs, "", syscall.MS_BIND, ""); err != nil {
 		return err
 	}
-	// remount rw
-	if err := syscall.Mount("", rootfs, "", syscall.MS_REMOUNT, ""); err != nil {
+	return nil
+}
+
+func prepareRW(path string) error {
+	// mount a tmpfs on tmp for upper and workdirs
+	// make it private as nothing else should be using this
+	tmp := filepath.Join(path, "tmp")
+	if err := syscall.Mount("tmpfs", tmp, "tmpfs", 0, "size=10%"); err != nil {
+		return err
+	}
+	// make it private as nothing else should be using this
+	if err := syscall.Mount("", tmp, "", syscall.MS_REMOUNT|syscall.MS_PRIVATE, ""); err != nil {
+		return err
+	}
+	upper := filepath.Join(tmp, "upper")
+	// make the mount points
+	if err := os.Mkdir(upper, 0744); err != nil {
+		return err
+	}
+	work := filepath.Join(tmp, "work")
+	if err := os.Mkdir(work, 0744); err != nil {
+		return err
+	}
+	lower := filepath.Join(path, "lower")
+	rootfs := filepath.Join(path, "rootfs")
+	opt := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower, upper, work)
+	if err := syscall.Mount("overlay", rootfs, "overlay", 0, opt); err != nil {
 		return err
 	}
 	return nil
 }
+
+// cleanup functions are best efforts only, mainly for rw onboot containers
+func cleanup(path string) {
+	// see if we are dealing with a read only or read write container
+	if _, err := os.Stat(filepath.Join(path, "lower")); err != nil {
+		cleanupRO(path)
+	} else {
+		cleanupRW(path)
+	}
+}
+
+func cleanupRO(path string) {
+	// remove the bind mount
+	rootfs := filepath.Join(path, "rootfs")
+	_ = syscall.Unmount(rootfs, 0)
+}
+
+func cleanupRW(path string) {
+	// remove the overlay mount
+	rootfs := filepath.Join(path, "rootfs")
+	_ = os.RemoveAll(rootfs)
+	_ = syscall.Unmount(rootfs, 0)
+	// remove the tmpfs
+	tmp := filepath.Join(path, "tmp")
+	_ = os.RemoveAll(tmp)
+	_ = syscall.Unmount(tmp, 0)
+}