Merge pull request #2811 from ijc/update-moby-tool

Bump moby tool to d9d2a91780

docs/vendoring.md

@@ -13,6 +13,12 @@ Details of usage of the `vndr` tool and the format of `vendor.conf` can be found
 Once done, you must run the `vndr` tool to add the necessary files to the `vendor` directory.
 The easiest way to do this is in a container.
 
+Currently if updating `github.com/moby/tool` it is also necessary to
+update `src/cmd/linuxkit/build.go` manually after updating `vendor.conf`:
+
+    hash=$(awk '/^github.com\/moby\/tool/ { print $2 }' src/cmd/linuxkit/vendor.conf)
+    curl -fsSL -o src/cmd/linuxkit/build.go https://raw.githubusercontent.com/moby/tool/${hash}/cmd/moby/build.go
+
 ## Updating in a container
 
 To update all dependencies:

src/cmd/linuxkit/build.go

@@ -185,14 +185,18 @@ func build(args []string) {
 		m.Trust = moby.TrustConfig{}
 	}
 
-	var buf *bytes.Buffer
+	var tf *os.File
 	var w io.Writer
 	if outputFile != nil {
 		w = outputFile
 	} else {
-		buf = new(bytes.Buffer)
+		if tf, err = ioutil.TempFile("", ""); err != nil {
-		w = buf
+			log.Fatalf("Error creating tempfile: %v", err)
+		}
+		defer os.Remove(tf.Name())
+		w = tf
 	}
+
 	// this is a weird interface, but currently only streamable types can have additional files
 	// need to split up the base tarball outputs from the secondary stages
 	var tp string
@@ -205,7 +209,11 @@ func build(args []string) {
 	}
 
 	if outputFile == nil {
-		image := buf.Bytes()
+		image := tf.Name()
+		if err := tf.Close(); err != nil {
+			log.Fatalf("Error closing tempfile: %v", err)
+		}
+
 		log.Infof("Create outputs:")
 		err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size)
 		if err != nil {

src/cmd/linuxkit/vendor.conf

@@ -24,7 +24,9 @@ github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
 github.com/mitchellh/go-ps 4fdf99ab29366514c69ccccddab5dc58b8d84062
 github.com/moby/datakit 97b3d230535397a813323902c23751e176481a86
 github.com/moby/hyperkit a12cd7250bcd8d689078e3e42ae4a7cf6a0cbaf3
-github.com/moby/tool 656bd87fd26b4cfc7da735939ce78cc7cb541181
+# When updating also:
+# curl -fsSL -o src/cmd/linuxkit/build.go https://raw.githubusercontent.com/moby/tool/«hash»/cmd/moby/build.go
+github.com/moby/tool d9d2a91780b34b92e669bbfa099f613bd9fad6bb
 github.com/moby/vpnkit 0e4293bb1058598c4b0a406ed171f52573ef414c
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448
 github.com/opencontainers/image-spec v1.0.0

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/docker.go

@@ -4,7 +4,6 @@ package moby
 // and also using the Docker API not shelling out
 
 import (
-	"bytes"
 	"errors"
 	"fmt"
 	"io"
@@ -81,25 +80,18 @@ func dockerCreate(image string) (string, error) {
 	return respBody.ID, nil
 }
 
-func dockerExport(container string) ([]byte, error) {
+func dockerExport(container string) (io.ReadCloser, error) {
 	log.Debugf("docker export: %s", container)
 	cli, err := dockerClient()
 	if err != nil {
-		return []byte{}, errors.New("could not initialize Docker API client")
+		return nil, errors.New("could not initialize Docker API client")
 	}
 	responseBody, err := cli.ContainerExport(context.Background(), container)
 	if err != nil {
-		return []byte{}, err
+		return nil, err
-	}
-	defer responseBody.Close()
-
-	output := bytes.NewBuffer(nil)
-	_, err = io.Copy(output, responseBody)
-	if err != nil {
-		return []byte{}, err
 	}
 
-	return output.Bytes(), nil
+	return responseBody, err
 }
 
 func dockerRm(container string) error {

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/image.go

@@ -82,7 +82,7 @@ func tarPrefix(path string, tw tarWriter) error {
 }
 
 // ImageTar takes a Docker image and outputs it to a tar stream
-func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) error {
+func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) (e error) {
 	log.Debugf("image tar: %s %s", ref, prefix)
 	if prefix != "" && prefix[len(prefix)-1] != byte('/') {
 		return fmt.Errorf("prefix does not end with /: %s", prefix)
@@ -119,15 +119,17 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull
 	if err != nil {
 		return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
 	}
-	err = dockerRm(container)
+	defer func() {
-	if err != nil {
+		contents.Close()
-		return fmt.Errorf("Failed to docker rm container %s: %v", container, err)
+
-	}
+		if err := dockerRm(container); e == nil && err != nil {
+			e = fmt.Errorf("Failed to docker rm container %s: %v", container, err)
+		}
+	}()
 
 	// now we need to filter out some files from the resulting tar archive
 
-	r := bytes.NewReader(contents)
+	tr := tar.NewReader(contents)
-	tr := tar.NewReader(r)
 
 	for {
 		hdr, err := tr.Next()

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/linuxkit.go

@@ -1,7 +1,6 @@
 package moby
 
 import (
-	"bytes"
 	"crypto/sha256"
 	"fmt"
 	"io"
@@ -18,13 +17,13 @@ kernel:
   image: linuxkit/kernel:4.9.39
   cmdline: "console=ttyS0"
 init:
-  - linuxkit/init:9250948d0de494df8a811edb3242b4584057cfe4
+  - linuxkit/init:00ab58c9681a0bf42b2e35134c1ccf1591ebb64d
-  - linuxkit/runc:abc3f292653e64a2fd488e9675ace19a55ec7023
+  - linuxkit/runc:f5960b83a8766ae083efc744fa63dbf877450e4f
 onboot:
   - name: mkimage
-    image: linuxkit/mkimage:e439d6108466186948ca7ea2a293fc6c1d1183fa
+    image: linuxkit/mkimage:a63b8ee4c5de335afc32ba850e0af319b25b96c0
   - name: poweroff
-    image: linuxkit/poweroff:bccfe1cb04fc7bb9f03613d2314f38abd2620f29
+    image: linuxkit/poweroff:3845c4d64d47a1ea367806be5547e44594b0fa91
 trust:
   org:
     - linuxkit
@@ -58,9 +57,21 @@ func ensureLinuxkitImage(name string) error {
 		return err
 	}
 	// TODO pass through --pull to here
-	buf := new(bytes.Buffer)
+	tf, err := ioutil.TempFile("", "")
-	Build(m, buf, false, "")
+	if err != nil {
-	image := buf.Bytes()
+		return err
+	}
+	defer os.Remove(tf.Name())
+	Build(m, tf, false, "")
+	if err := tf.Close(); err != nil {
+		return err
+	}
+
+	image, err := os.Open(tf.Name())
+	if err != nil {
+		return err
+	}
+	defer image.Close()
 	kernel, initrd, cmdline, err := tarToInitrd(image)
 	if err != nil {
 		return fmt.Errorf("Error converting to initrd: %v", err)

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/output.go

@@ -4,6 +4,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"runtime"
@@ -24,8 +25,8 @@ const (
 	rpi3       = "linuxkit/mkimage-rpi3:0735656fff247ca978135e3aeb62864adc612180"
 )
 
-var outFuns = map[string]func(string, []byte, int) error{
+var outFuns = map[string]func(string, io.Reader, int) error{
-	"kernel+initrd": func(base string, image []byte, size int) error {
+	"kernel+initrd": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -36,7 +37,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"tar-kernel-initrd": func(base string, image []byte, size int) error {
+	"tar-kernel-initrd": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -46,21 +47,21 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"iso-bios": func(base string, image []byte, size int) error {
+	"iso-bios": func(base string, image io.Reader, size int) error {
 		err := outputIso(isoBios, base+".iso", image)
 		if err != nil {
 			return fmt.Errorf("Error writing iso-bios output: %v", err)
 		}
 		return nil
 	},
-	"iso-efi": func(base string, image []byte, size int) error {
+	"iso-efi": func(base string, image io.Reader, size int) error {
 		err := outputIso(isoEfi, base+"-efi.iso", image)
 		if err != nil {
 			return fmt.Errorf("Error writing iso-efi output: %v", err)
 		}
 		return nil
 	},
-	"raw-bios": func(base string, image []byte, size int) error {
+	"raw-bios": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -71,7 +72,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"raw-efi": func(base string, image []byte, size int) error {
+	"raw-efi": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -82,7 +83,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"aws": func(base string, image []byte, size int) error {
+	"aws": func(base string, image io.Reader, size int) error {
 		filename := base + ".raw"
 		log.Infof("  %s", filename)
 		kernel, initrd, cmdline, err := tarToInitrd(image)
@@ -95,7 +96,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"gcp": func(base string, image []byte, size int) error {
+	"gcp": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -106,7 +107,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"qcow2-bios": func(base string, image []byte, size int) error {
+	"qcow2-bios": func(base string, image io.Reader, size int) error {
 		filename := base + ".qcow2"
 		log.Infof("  %s", filename)
 		kernel, initrd, cmdline, err := tarToInitrd(image)
@@ -119,7 +120,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"vhd": func(base string, image []byte, size int) error {
+	"vhd": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -130,7 +131,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"dynamic-vhd": func(base string, image []byte, size int) error {
+	"dynamic-vhd": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -141,7 +142,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"vmdk": func(base string, image []byte, size int) error {
+	"vmdk": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -152,7 +153,7 @@ var outFuns = map[string]func(string, []byte, int) error{
 		}
 		return nil
 	},
-	"rpi3": func(base string, image []byte, size int) error {
+	"rpi3": func(base string, image io.Reader, size int) error {
 		if runtime.GOARCH != "arm64" {
 			return fmt.Errorf("Raspberry Pi output currently only supported on arm64")
 		}
@@ -197,7 +198,7 @@ func ValidateFormats(formats []string) error {
 }
 
 // Formats generates all the specified output formats
-func Formats(base string, image []byte, formats []string, size int) error {
+func Formats(base string, image string, formats []string, size int) error {
 	log.Debugf("format: %v %s", formats, base)
 
 	err := ValidateFormats(formats)
@@ -205,20 +206,23 @@ func Formats(base string, image []byte, formats []string, size int) error {
 		return err
 	}
 	for _, o := range formats {
-		f := outFuns[o]
+		ir, err := os.Open(image)
-		err := f(base, image, size)
 		if err != nil {
 			return err
 		}
+		defer ir.Close()
+		f := outFuns[o]
+		if err := f(base, ir, size); err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
-func tarToInitrd(image []byte) ([]byte, []byte, string, error) {
+func tarToInitrd(r io.Reader) ([]byte, []byte, string, error) {
 	w := new(bytes.Buffer)
 	iw := initrd.NewWriter(w)
-	r := bytes.NewReader(image)
 	tr := tar.NewReader(r)
 	kernel, cmdline, err := initrd.CopySplitTar(iw, tr)
 	if err != nil {
@@ -288,7 +292,7 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str
 	return dockerRun(buf, output, true, image, cmdline)
 }
 
-func outputIso(image, filename string, filesystem []byte) error {
+func outputIso(image, filename string, filesystem io.Reader) error {
 	log.Debugf("output ISO: %s %s", image, filename)
 	log.Infof("  %s", filename)
 	output, err := os.Create(filename)
@@ -296,10 +300,10 @@ func outputIso(image, filename string, filesystem []byte) error {
 		return err
 	}
 	defer output.Close()
-	return dockerRun(bytes.NewBuffer(filesystem), output, true, image)
+	return dockerRun(filesystem, output, true, image)
 }
 
-func outputRPi3(image, filename string, filesystem []byte) error {
+func outputRPi3(image, filename string, filesystem io.Reader) error {
 	log.Debugf("output RPi3: %s %s", image, filename)
 	log.Infof("  %s", filename)
 	output, err := os.Create(filename)
@@ -307,7 +311,7 @@ func outputRPi3(image, filename string, filesystem []byte) error {
 		return err
 	}
 	defer output.Close()
-	return dockerRun(bytes.NewBuffer(filesystem), output, true, image)
+	return dockerRun(filesystem, output, true, image)
 }
 
 func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline string) error {