Merge pull request #1370 from justincormack/no-docker2tar

Remove docker2tar docker image, use Docker directly from Go

Makefile

@@ -13,6 +13,7 @@ endif
 
 bin/moby: $(MOBY_DEPS) | bin
 	tar cf - vendor src/initrd src/pad4 -C src/cmd/moby . | docker run --rm --net=none --log-driver=none -i $(CROSS) $(GO_COMPILE) --package github.com/docker/moby -o $@ | tar xf -
+	touch $@
 
 moby-initrd.img: bin/moby moby.yaml
 	bin/moby build moby.yaml

src/cmd/moby/build.go

@@ -12,10 +12,6 @@ import (
 	"github.com/docker/moby/src/initrd"
 )
 
-const (
-	docker2tar = "mobylinux/docker2tar:82a3f11f70b2959c7100dd6e184b511ebfc65908@sha256:e4fd36febc108477a2e5316d263ac257527779409891c7ac10d455a162df05c1"
-)
-
 func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
 	tr := tar.NewReader(buf)
 
@@ -112,7 +108,7 @@ func build(name string, args []string) {
 	containers = append(containers, ktar)
 
 	// convert init image to tarball
-	init, err := dockerRun("-v", "/var/run/docker.sock:/var/run/docker.sock", docker2tar, m.Init)
+	init, err := imageExtract(m.Init)
 	if err != nil {
 		log.Fatalf("Failed to build init tarball: %v", err)
 	}

src/cmd/moby/image.go

@@ -0,0 +1,213 @@
+package main
+
+import (
+	"archive/tar"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+)
+
+// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we
+// used the containerd libraries to do this instead locally direct from a local image
+// cache as it would be much simpler.
+
+func dockerCreate(image string) (string, error) {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return "", errors.New("Docker does not seem to be installed")
+	}
+	// we do not ever run the container, so /dev/null is used as command
+	args := []string{"create", image, "/dev/null"}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return "", err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return "", err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return "", err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return "", err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return "", err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return "", fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	container := strings.TrimSpace(string(stdout))
+	return container, nil
+}
+
+func dockerExport(container string) ([]byte, error) {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return []byte{}, errors.New("Docker does not seem to be installed")
+	}
+	args := []string{"export", container}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stdout, err := ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return []byte{}, err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return []byte{}, fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	return stdout, nil
+}
+
+func dockerRm(container string) error {
+	docker, err := exec.LookPath("docker")
+	if err != nil {
+		return errors.New("Docker does not seem to be installed")
+	}
+	args := []string{"rm", container}
+	cmd := exec.Command(docker, args...)
+
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return err
+	}
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		return err
+	}
+
+	_, err = ioutil.ReadAll(stdoutPipe)
+	if err != nil {
+		return err
+	}
+
+	stderr, err := ioutil.ReadAll(stderrPipe)
+	if err != nil {
+		return err
+	}
+
+	err = cmd.Wait()
+	if err != nil {
+		return fmt.Errorf("%s: %s", err, stderr)
+	}
+
+	return nil
+}
+
+var exclude = map[string]bool{
+	".dockerenv":  true,
+	"Dockerfile":  true,
+	"dev/console": true,
+	"dev/pts":     true,
+	"dev/shm":     true,
+}
+
+var replace = map[string]string{
+	"etc/hosts": `127.0.0.1       localhost
+::1     localhost ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+`,
+	"etc/resolv.conf": `nameserver 8.8.8.8
+nameserver 8.8.4.4
+nameserver 2001:4860:4860::8888
+nameserver 2001:4860:4860::8844
+`,
+	"etc/hostname": "moby",
+}
+
+func imageExtract(image string) ([]byte, error) {
+	container, err := dockerCreate(image)
+	if err != nil {
+		return []byte{}, fmt.Errorf("Failed to docker create image %s: %v", image, err)
+	}
+	contents, err := dockerExport(container)
+	if err != nil {
+		return []byte{}, fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
+	}
+	err = dockerRm(container)
+	if err != nil {
+		return []byte{}, fmt.Errorf("Failed to docker rm container %s: %v", container, err)
+	}
+
+	// now we need to filter out some files from the resulting tar archive
+	out := new(bytes.Buffer)
+	tw := tar.NewWriter(out)
+
+	r := bytes.NewReader(contents)
+	tr := tar.NewReader(r)
+
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return []byte{}, err
+		}
+		if exclude[hdr.Name] {
+			io.Copy(ioutil.Discard, tr)
+		} else if replace[hdr.Name] != "" {
+			hdr.Size = int64(len(replace[hdr.Name]))
+			tw.WriteHeader(hdr)
+			buf := bytes.NewBufferString(replace[hdr.Name])
+			io.Copy(tw, buf)
+		} else {
+			tw.WriteHeader(hdr)
+			io.Copy(tw, tr)
+		}
+	}
+	err = tw.Close()
+	if err != nil {
+		return []byte{}, err
+	}
+	return out.Bytes(), nil
+}

tools/docker2tar/Dockerfile

@@ -1,5 +0,0 @@
-FROM docker:1.13.1
-
-COPY . ./
-
-ENTRYPOINT ["/docker2tar.sh"]

tools/docker2tar/Makefile

@@ -1,29 +0,0 @@
-.PHONY: tag push
-
-BASE=docker:1.13.1
-IMAGE=docker2tar
-
-default: push
-
-hash: Dockerfile docker2tar.sh
-	DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
-	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
-	docker run --entrypoint=/bin/sh --rm $(IMAGE):build -c 'cat $^ /usr/local/bin/* /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@
-
-push: hash
-	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
-		 (docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
-		  docker push mobylinux/$(IMAGE):$(shell cat hash))
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-tag: hash
-	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
-		docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
-	docker rmi $(IMAGE):build
-	rm -f hash
-
-clean:
-	rm -f hash
-
-.DELETE_ON_ERROR:

tools/docker2tar/docker2tar.sh

@@ -1,43 +0,0 @@
-#!/bin/sh
-
-set -e
-
-# arguments is image name
-
-IMAGE="$1"; shift
-
-cd /tmp
-
-# extract rootfs
-EXCLUDE="--exclude .dockerenv --exclude Dockerfile \
-        --exclude dev/console --exclude dev/pts --exclude dev/shm \
-        --exclude etc/hostname --exclude etc/hosts --exclude etc/mtab --exclude etc/resolv.conf"
-CONTAINER="$(docker create $IMAGE /dev/null)"
-docker export "$CONTAINER" | tar -xf - $EXCLUDE
-docker rm "$CONTAINER" > /dev/null
-
-# these three files are bind mounted in by docker so they are not what we want
-
-mkdir -p etc
-
-cat << EOF > etc/hosts
-127.0.0.1       localhost
-::1     localhost ip6-localhost ip6-loopback
-fe00::0 ip6-localnet
-ff00::0 ip6-mcastprefix
-ff02::1 ip6-allnodes
-ff02::2 ip6-allrouters
-EOF
-
-cat << EOF > etc/resolv.conf
-nameserver 8.8.8.8
-nameserver 8.8.4.4
-nameserver 2001:4860:4860::8888
-nameserver 2001:4860:4860::8844
-EOF
-
-printf 'moby' > etc/hostname
-
-ln -s /proc/mounts etc/mtab
-
-tar cf - .