Use Go code to extract rootfs from system containers

- this removes the use of riddler to extract the rootfs, use code
  we were using for rootfs. riddler now just geenrates the config,
  next stage is to generate this ourselves
- change the naming of the daemons so no longer include number as we
  do not guarantee ordering as they start up simultaneously

Signed-off-by: Justin Cormack <justin.cormack@docker.com>

src/cmd/moby/build.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"log"
@@ -108,7 +109,7 @@ func build(name string, args []string) {
 	containers = append(containers, ktar)
 
 	// convert init image to tarball
-	init, err := imageExtract(m.Init)
+	init, err := ImageExtract(m.Init, "")
 	if err != nil {
 		log.Fatalf("Failed to build init tarball: %v", err)
 	}
@@ -116,20 +117,29 @@ func build(name string, args []string) {
 	containers = append(containers, buffer)
 
 	for i, image := range m.System {
-		args := ConfigToRun(i, "system", &image)
+		config, err := ConfigToOCI(&image)
-		out, err := dockerRun(args...)
 		if err != nil {
-			log.Fatalf("Failed to build container tarball: %v", err)
+			log.Fatalf("Failed to run riddler to get config.json for %s: %v", image.Image, err)
+		}
+		so := fmt.Sprintf("%03d", i)
+		path := "containers/system/" + so + "-" + image.Name
+		out, err := ImageBundle(path, image.Image, config)
+		if err != nil {
+			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
 		}
 		buffer := bytes.NewBuffer(out)
 		containers = append(containers, buffer)
 	}
 
-	for i, image := range m.Daemon {
+	for _, image := range m.Daemon {
-		args := ConfigToRun(i, "daemon", &image)
+		config, err := ConfigToOCI(&image)
-		out, err := dockerRun(args...)
 		if err != nil {
-			log.Fatalf("Failed to build container tarball: %v", err)
+			log.Fatalf("Failed to run riddler to get config.json for %s: %v", image.Image, err)
+		}
+		path := "containers/daemon/" + image.Name
+		out, err := ImageBundle(path, image.Image, config)
+		if err != nil {
+			log.Fatalf("Failed to extract root filesystem for %s: %v", image.Image, err)
 		}
 		buffer := bytes.NewBuffer(out)
 		containers = append(containers, buffer)

src/cmd/moby/config.go

@@ -50,7 +50,7 @@ type MobyImage struct {
 	ReadOnly     bool `yaml:"read_only"`
 }
 
-const riddler = "mobylinux/riddler:c23ab4b6e2a2a4ebd4dd51a059cef7f270da72cb@sha256:7e7744b2f554518411633200db98e599782b120e323348495f43f540de26f7b6"
+const riddler = "mobylinux/riddler:2b4051422b155f659019f9e3fef8cca04e153f5c@sha256:f4bb0c39f1e5c636ed52ebd3ed8ec447ca6c0dc554ffb5784cbeff423ac70d34"
 
 // NewConfig parses a config file
 func NewConfig(config []byte) (*Moby, error) {
@@ -64,11 +64,10 @@ func NewConfig(config []byte) (*Moby, error) {
 	return &m, nil
 }
 
-// ConfigToRun converts a config to a series of arguments for docker run
+// ConfigToOCI converts a config specification to an OCI config file
-func ConfigToRun(order int, path string, image *MobyImage) []string {
+func ConfigToOCI(image *MobyImage) (string, error) {
 	// riddler arguments
-	so := fmt.Sprintf("%03d", order)
+	args := []string{"-v", "/var/run/docker.sock:/var/run/docker.sock", riddler, image.Image}
-	args := []string{"-v", "/var/run/docker.sock:/var/run/docker.sock", riddler, image.Image, "/containers/" + path + "/" + so + "-" + image.Name}
 	// docker arguments
 	args = append(args, "--cap-drop", "all")
 	for _, cap := range image.Capabilities {
@@ -107,7 +106,12 @@ func ConfigToRun(order int, path string, image *MobyImage) []string {
 	// command
 	args = append(args, image.Command...)
 
-	return args
+	config, err := dockerRun(args...)
+	if err != nil {
+		return "", fmt.Errorf("Failed to run riddler to get config.json: %v", err)
+	}
+
+	return string(config), nil
 }
 
 func filesystem(m *Moby) (*bytes.Buffer, error) {

src/cmd/moby/image.go

@@ -164,23 +164,69 @@ nameserver 2001:4860:4860::8844
 	"etc/hostname": "moby",
 }
 
-func imageExtract(image string) ([]byte, error) {
+// ImageExtract extracts the filesystem from an image and returns a tarball with the files prefixed by the given path
+func ImageExtract(image, prefix string) ([]byte, error) {
+	out := new(bytes.Buffer)
+	tw := tar.NewWriter(out)
+	err := tarPrefix(prefix, tw)
+	if err != nil {
+		return []byte{}, err
+	}
+	err = imageTar(image, prefix, tw)
+	if err != nil {
+		return []byte{}, err
+	}
+	err = tw.Close()
+	if err != nil {
+		return []byte{}, err
+	}
+	return out.Bytes(), nil
+}
+
+// tarPrefix creates the leading directories for a path
+func tarPrefix(path string, tw *tar.Writer) error {
+	if path == "" {
+		return nil
+	}
+	if path[len(path)-1] != byte('/') {
+		return fmt.Errorf("path does not end with /: %s", path)
+	}
+	path = path[:len(path)-1]
+	if path[0] == byte('/') {
+		return fmt.Errorf("path should be relative: %s", path)
+	}
+	mkdir := ""
+	for _, dir := range strings.Split(path, "/") {
+		mkdir = mkdir + dir
+		hdr := &tar.Header{
+			Name:     mkdir,
+			Mode:     0755,
+			Typeflag: tar.TypeDir,
+		}
+		tw.WriteHeader(hdr)
+		mkdir = mkdir + "/"
+	}
+	return nil
+}
+
+func imageTar(image, prefix string, tw *tar.Writer) error {
+	if prefix != "" && prefix[len(prefix)-1] != byte('/') {
+		return fmt.Errorf("prefix does not end with /: %s", prefix)
+	}
 	container, err := dockerCreate(image)
 	if err != nil {
-		return []byte{}, fmt.Errorf("Failed to docker create image %s: %v", image, err)
+		return fmt.Errorf("Failed to docker create image %s: %v", image, err)
 	}
 	contents, err := dockerExport(container)
 	if err != nil {
-		return []byte{}, fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
+		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
 	}
 	err = dockerRm(container)
 	if err != nil {
-		return []byte{}, fmt.Errorf("Failed to docker rm container %s: %v", container, err)
+		return fmt.Errorf("Failed to docker rm container %s: %v", container, err)
 	}
 
 	// now we need to filter out some files from the resulting tar archive
-	out := new(bytes.Buffer)
-	tw := tar.NewWriter(out)
 
 	r := bytes.NewReader(contents)
 	tr := tar.NewReader(r)
@@ -191,21 +237,58 @@ func imageExtract(image string) ([]byte, error) {
 			break
 		}
 		if err != nil {
-			return []byte{}, err
+			return err
 		}
 		if exclude[hdr.Name] {
 			io.Copy(ioutil.Discard, tr)
 		} else if replace[hdr.Name] != "" {
-			hdr.Size = int64(len(replace[hdr.Name]))
+			contents := replace[hdr.Name]
+			hdr.Size = int64(len(contents))
+			hdr.Name = prefix + hdr.Name
 			tw.WriteHeader(hdr)
-			buf := bytes.NewBufferString(replace[hdr.Name])
+			buf := bytes.NewBufferString(contents)
 			io.Copy(tw, buf)
+			io.Copy(ioutil.Discard, tr)
 		} else {
+			hdr.Name = prefix + hdr.Name
 			tw.WriteHeader(hdr)
 			io.Copy(tw, tr)
 		}
 	}
 	err = tw.Close()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// ImageBundle produces an OCI bundle at the given path in a tarball, given an image and a config.json
+func ImageBundle(path, image, config string) ([]byte, error) {
+	out := new(bytes.Buffer)
+	tw := tar.NewWriter(out)
+	err := tarPrefix(path+"/rootfs/", tw)
+	if err != nil {
+		return []byte{}, err
+	}
+	hdr := &tar.Header{
+		Name: path + "/" + "config.json",
+		Mode: 0644,
+		Size: int64(len(config)),
+	}
+	err = tw.WriteHeader(hdr)
+	if err != nil {
+		return []byte{}, err
+	}
+	buf := bytes.NewBufferString(config)
+	_, err = io.Copy(tw, buf)
+	if err != nil {
+		return []byte{}, err
+	}
+	err = imageTar(image, path+"/rootfs/", tw)
+	if err != nil {
+		return []byte{}, err
+	}
+	err = tw.Close()
 	if err != nil {
 		return []byte{}, err
 	}

tools/riddler/riddler.sh

@@ -3,15 +3,12 @@
 set -e
 
 # arguments are image name, prefix, then arguments passed to Docker
-# eg ./riddler.sh alpine:3.4 / --read-only alpine:3.4 ls
+# eg ./riddler.sh alpine:3.4 --read-only alpine:3.4 ls
-# This script will output a tarball under prefix/ with rootfs and config.json
+# This script will output config.json
 
 IMAGE="$1"; shift
-PREFIX="$1"; shift
 
 cd /tmp
-mkdir -p /tmp/$PREFIX
-cd /tmp/$PREFIX
 
 # riddler always adds the apparmor options if this is not present
 EXTRA_OPTIONS="--security-opt apparmor=unconfined"
@@ -22,7 +19,7 @@ riddler $CONTAINER > /dev/null
 docker rm $CONTAINER > /dev/null
 
 # unfixed known issues
-# noNewPrivileges is always set by riddler, but that is fine for our use cases
+# noNewPrivileges is always set by riddler
 
 # These fixes should be removed when riddler is fixed
 # process.rlimits, just a constant at present, not useful
@@ -47,16 +44,5 @@ cat config.json.orig | \
   jq 'if .root.readonly==true then .mounts = (.mounts|map(if .destination=="/dev" then .options |= .+ ["ro"] else . end)) else . end' | \
   jq '.mounts = if .process.capabilities | length != 38 then (.mounts|map(if .destination=="/sys" then .options |= .+ ["ro"] else . end)) else . end' \
   > config.json
-rm config.json.orig
 
-# extract rootfs
+cat config.json
-EXCLUDE="--exclude .dockerenv --exclude Dockerfile \
-        --exclude dev/console --exclude dev/pts --exclude dev/shm \
-        --exclude etc/hostname --exclude etc/hosts --exclude etc/resolv.conf"
-mkdir -p rootfs
-CONTAINER="$(docker create $IMAGE /dev/null)"
-docker export "$CONTAINER" | tar -xf - -C rootfs $EXCLUDE
-docker rm "$CONTAINER" > /dev/null
-
-cd /tmp
-tar cf - .