From c76074ff661ef7cadc15041db7babb54085eb850 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Wed, 22 Mar 2017 22:24:14 +0000 Subject: [PATCH] Remove docker2tar docker image, use Docker directly from Go Removing the left over indirect creates that use the Docker socket and run in containers not directly. See #1347 Signed-off-by: Justin Cormack --- Makefile | 1 + src/cmd/moby/build.go | 6 +- src/cmd/moby/image.go | 213 +++++++++++++++++++++++++++++++++ tools/docker2tar/Dockerfile | 5 - tools/docker2tar/Makefile | 29 ----- tools/docker2tar/docker2tar.sh | 43 ------- 6 files changed, 215 insertions(+), 82 deletions(-) create mode 100644 src/cmd/moby/image.go delete mode 100644 tools/docker2tar/Dockerfile delete mode 100644 tools/docker2tar/Makefile delete mode 100755 tools/docker2tar/docker2tar.sh diff --git a/Makefile b/Makefile index 4ae75d47c..b0f046617 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ endif bin/moby: $(MOBY_DEPS) | bin tar cf - vendor src/initrd src/pad4 -C src/cmd/moby . | docker run --rm --net=none --log-driver=none -i $(CROSS) $(GO_COMPILE) --package github.com/docker/moby -o $@ | tar xf - + touch $@ moby-initrd.img: bin/moby moby.yaml bin/moby build moby.yaml diff --git a/src/cmd/moby/build.go b/src/cmd/moby/build.go index 7bf2449d0..3f67799d6 100644 --- a/src/cmd/moby/build.go +++ b/src/cmd/moby/build.go @@ -12,10 +12,6 @@ import ( "github.com/docker/moby/src/initrd" ) -const ( - docker2tar = "mobylinux/docker2tar:82a3f11f70b2959c7100dd6e184b511ebfc65908@sha256:e4fd36febc108477a2e5316d263ac257527779409891c7ac10d455a162df05c1" -) - func untarKernel(buf *bytes.Buffer, bzimageName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) { tr := tar.NewReader(buf) @@ -112,7 +108,7 @@ func build(name string, args []string) { containers = append(containers, ktar) // convert init image to tarball - init, err := dockerRun("-v", "/var/run/docker.sock:/var/run/docker.sock", docker2tar, m.Init) + init, err := imageExtract(m.Init) if err != nil { log.Fatalf("Failed to build init tarball: %v", err) } diff --git a/src/cmd/moby/image.go b/src/cmd/moby/image.go new file mode 100644 index 000000000..c0531a872 --- /dev/null +++ b/src/cmd/moby/image.go @@ -0,0 +1,213 @@ +package main + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "os/exec" + "strings" +) + +// This uses Docker to convert a Docker image into a tarball. It would be an improvement if we +// used the containerd libraries to do this instead locally direct from a local image +// cache as it would be much simpler. + +func dockerCreate(image string) (string, error) { + docker, err := exec.LookPath("docker") + if err != nil { + return "", errors.New("Docker does not seem to be installed") + } + // we do not ever run the container, so /dev/null is used as command + args := []string{"create", image, "/dev/null"} + cmd := exec.Command(docker, args...) + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return "", err + } + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return "", err + } + + err = cmd.Start() + if err != nil { + return "", err + } + + stdout, err := ioutil.ReadAll(stdoutPipe) + if err != nil { + return "", err + } + + stderr, err := ioutil.ReadAll(stderrPipe) + if err != nil { + return "", err + } + + err = cmd.Wait() + if err != nil { + return "", fmt.Errorf("%s: %s", err, stderr) + } + + container := strings.TrimSpace(string(stdout)) + return container, nil +} + +func dockerExport(container string) ([]byte, error) { + docker, err := exec.LookPath("docker") + if err != nil { + return []byte{}, errors.New("Docker does not seem to be installed") + } + args := []string{"export", container} + cmd := exec.Command(docker, args...) + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return []byte{}, err + } + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return []byte{}, err + } + + err = cmd.Start() + if err != nil { + return []byte{}, err + } + + stdout, err := ioutil.ReadAll(stdoutPipe) + if err != nil { + return []byte{}, err + } + + stderr, err := ioutil.ReadAll(stderrPipe) + if err != nil { + return []byte{}, err + } + + err = cmd.Wait() + if err != nil { + return []byte{}, fmt.Errorf("%s: %s", err, stderr) + } + + return stdout, nil +} + +func dockerRm(container string) error { + docker, err := exec.LookPath("docker") + if err != nil { + return errors.New("Docker does not seem to be installed") + } + args := []string{"rm", container} + cmd := exec.Command(docker, args...) + + stderrPipe, err := cmd.StderrPipe() + if err != nil { + return err + } + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + return err + } + + err = cmd.Start() + if err != nil { + return err + } + + _, err = ioutil.ReadAll(stdoutPipe) + if err != nil { + return err + } + + stderr, err := ioutil.ReadAll(stderrPipe) + if err != nil { + return err + } + + err = cmd.Wait() + if err != nil { + return fmt.Errorf("%s: %s", err, stderr) + } + + return nil +} + +var exclude = map[string]bool{ + ".dockerenv": true, + "Dockerfile": true, + "dev/console": true, + "dev/pts": true, + "dev/shm": true, +} + +var replace = map[string]string{ + "etc/hosts": `127.0.0.1 localhost +::1 localhost ip6-localhost ip6-loopback +fe00::0 ip6-localnet +ff00::0 ip6-mcastprefix +ff02::1 ip6-allnodes +ff02::2 ip6-allrouters +`, + "etc/resolv.conf": `nameserver 8.8.8.8 +nameserver 8.8.4.4 +nameserver 2001:4860:4860::8888 +nameserver 2001:4860:4860::8844 +`, + "etc/hostname": "moby", +} + +func imageExtract(image string) ([]byte, error) { + container, err := dockerCreate(image) + if err != nil { + return []byte{}, fmt.Errorf("Failed to docker create image %s: %v", image, err) + } + contents, err := dockerExport(container) + if err != nil { + return []byte{}, fmt.Errorf("Failed to docker export container from container %s: %v", container, err) + } + err = dockerRm(container) + if err != nil { + return []byte{}, fmt.Errorf("Failed to docker rm container %s: %v", container, err) + } + + // now we need to filter out some files from the resulting tar archive + out := new(bytes.Buffer) + tw := tar.NewWriter(out) + + r := bytes.NewReader(contents) + tr := tar.NewReader(r) + + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return []byte{}, err + } + if exclude[hdr.Name] { + io.Copy(ioutil.Discard, tr) + } else if replace[hdr.Name] != "" { + hdr.Size = int64(len(replace[hdr.Name])) + tw.WriteHeader(hdr) + buf := bytes.NewBufferString(replace[hdr.Name]) + io.Copy(tw, buf) + } else { + tw.WriteHeader(hdr) + io.Copy(tw, tr) + } + } + err = tw.Close() + if err != nil { + return []byte{}, err + } + return out.Bytes(), nil +} diff --git a/tools/docker2tar/Dockerfile b/tools/docker2tar/Dockerfile deleted file mode 100644 index 5c0398172..000000000 --- a/tools/docker2tar/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM docker:1.13.1 - -COPY . ./ - -ENTRYPOINT ["/docker2tar.sh"] diff --git a/tools/docker2tar/Makefile b/tools/docker2tar/Makefile deleted file mode 100644 index d56d12873..000000000 --- a/tools/docker2tar/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -.PHONY: tag push - -BASE=docker:1.13.1 -IMAGE=docker2tar - -default: push - -hash: Dockerfile docker2tar.sh - DOCKER_CONTENT_TRUST=1 docker pull $(BASE) - tar cf - $^ | docker build --no-cache -t $(IMAGE):build - - docker run --entrypoint=/bin/sh --rm $(IMAGE):build -c 'cat $^ /usr/local/bin/* /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@ - -push: hash - docker pull mobylinux/$(IMAGE):$(shell cat hash) || \ - (docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \ - docker push mobylinux/$(IMAGE):$(shell cat hash)) - docker rmi $(IMAGE):build - rm -f hash - -tag: hash - docker pull mobylinux/$(IMAGE):$(shell cat hash) || \ - docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) - docker rmi $(IMAGE):build - rm -f hash - -clean: - rm -f hash - -.DELETE_ON_ERROR: diff --git a/tools/docker2tar/docker2tar.sh b/tools/docker2tar/docker2tar.sh deleted file mode 100755 index 052be90b0..000000000 --- a/tools/docker2tar/docker2tar.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh - -set -e - -# arguments is image name - -IMAGE="$1"; shift - -cd /tmp - -# extract rootfs -EXCLUDE="--exclude .dockerenv --exclude Dockerfile \ - --exclude dev/console --exclude dev/pts --exclude dev/shm \ - --exclude etc/hostname --exclude etc/hosts --exclude etc/mtab --exclude etc/resolv.conf" -CONTAINER="$(docker create $IMAGE /dev/null)" -docker export "$CONTAINER" | tar -xf - $EXCLUDE -docker rm "$CONTAINER" > /dev/null - -# these three files are bind mounted in by docker so they are not what we want - -mkdir -p etc - -cat << EOF > etc/hosts -127.0.0.1 localhost -::1 localhost ip6-localhost ip6-loopback -fe00::0 ip6-localnet -ff00::0 ip6-mcastprefix -ff02::1 ip6-allnodes -ff02::2 ip6-allrouters -EOF - -cat << EOF > etc/resolv.conf -nameserver 8.8.8.8 -nameserver 8.8.4.4 -nameserver 2001:4860:4860::8888 -nameserver 2001:4860:4860::8844 -EOF - -printf 'moby' > etc/hostname - -ln -s /proc/mounts etc/mtab - -tar cf - .