From dd799eb2302a5140e7d8fc33b081406235471650 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 13 Dec 2017 10:16:45 +0000 Subject: [PATCH] Bump moby tool to d9d2a91780b3 https://github.com/moby/tool/compare/656bd87fd26b...d9d2a91780b3 d9d2a91 Merge pull request #193 from ijc/bugfix-191 307f13b Defer dockerRm until we are finished with the contents ebd7228 Merge pull request #191 from ijc/reduce-memory-via-tempfiles 3045a80 Stream `docker export` directly to consumer 9f44acf Generate intermediate image into a temp file 9558740 Add cpu and mem profiling options Reduces the memory usage substantially. While here make some notes about the need to update src/cmd/linuxkit/build.go where people might see them. Signed-off-by: Ian Campbell --- docs/vendoring.md | 6 +++ src/cmd/linuxkit/build.go | 16 ++++-- src/cmd/linuxkit/vendor.conf | 4 +- .../github.com/moby/tool/src/moby/docker.go | 16 ++---- .../github.com/moby/tool/src/moby/image.go | 16 +++--- .../github.com/moby/tool/src/moby/linuxkit.go | 27 +++++++--- .../github.com/moby/tool/src/moby/output.go | 50 ++++++++++--------- 7 files changed, 80 insertions(+), 55 deletions(-) diff --git a/docs/vendoring.md b/docs/vendoring.md index 864459578..4df04b92a 100644 --- a/docs/vendoring.md +++ b/docs/vendoring.md @@ -13,6 +13,12 @@ Details of usage of the `vndr` tool and the format of `vendor.conf` can be found Once done, you must run the `vndr` tool to add the necessary files to the `vendor` directory. The easiest way to do this is in a container. +Currently if updating `github.com/moby/tool` it is also necessary to +update `src/cmd/linuxkit/build.go` manually after updating `vendor.conf`: + + hash=$(awk '/^github.com\/moby\/tool/ { print $2 }' src/cmd/linuxkit/vendor.conf) + curl -fsSL -o src/cmd/linuxkit/build.go https://raw.githubusercontent.com/moby/tool/${hash}/cmd/moby/build.go + ## Updating in a container To update all dependencies: diff --git a/src/cmd/linuxkit/build.go b/src/cmd/linuxkit/build.go index dd67b8049..ef8f623e0 100644 --- a/src/cmd/linuxkit/build.go +++ b/src/cmd/linuxkit/build.go @@ -185,14 +185,18 @@ func build(args []string) { m.Trust = moby.TrustConfig{} } - var buf *bytes.Buffer + var tf *os.File var w io.Writer if outputFile != nil { w = outputFile } else { - buf = new(bytes.Buffer) - w = buf + if tf, err = ioutil.TempFile("", ""); err != nil { + log.Fatalf("Error creating tempfile: %v", err) + } + defer os.Remove(tf.Name()) + w = tf } + // this is a weird interface, but currently only streamable types can have additional files // need to split up the base tarball outputs from the secondary stages var tp string @@ -205,7 +209,11 @@ func build(args []string) { } if outputFile == nil { - image := buf.Bytes() + image := tf.Name() + if err := tf.Close(); err != nil { + log.Fatalf("Error closing tempfile: %v", err) + } + log.Infof("Create outputs:") err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size) if err != nil { diff --git a/src/cmd/linuxkit/vendor.conf b/src/cmd/linuxkit/vendor.conf index 23eb34dd6..c6c430b1d 100644 --- a/src/cmd/linuxkit/vendor.conf +++ b/src/cmd/linuxkit/vendor.conf @@ -24,7 +24,9 @@ github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d github.com/mitchellh/go-ps 4fdf99ab29366514c69ccccddab5dc58b8d84062 github.com/moby/datakit 97b3d230535397a813323902c23751e176481a86 github.com/moby/hyperkit a12cd7250bcd8d689078e3e42ae4a7cf6a0cbaf3 -github.com/moby/tool 656bd87fd26b4cfc7da735939ce78cc7cb541181 +# When updating also: +# curl -fsSL -o src/cmd/linuxkit/build.go https://raw.githubusercontent.com/moby/tool/«hash»/cmd/moby/build.go +github.com/moby/tool d9d2a91780b34b92e669bbfa099f613bd9fad6bb github.com/moby/vpnkit 0e4293bb1058598c4b0a406ed171f52573ef414c github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448 github.com/opencontainers/image-spec v1.0.0 diff --git a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/docker.go b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/docker.go index 640cf8119..296a61108 100644 --- a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/docker.go +++ b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/docker.go @@ -4,7 +4,6 @@ package moby // and also using the Docker API not shelling out import ( - "bytes" "errors" "fmt" "io" @@ -81,25 +80,18 @@ func dockerCreate(image string) (string, error) { return respBody.ID, nil } -func dockerExport(container string) ([]byte, error) { +func dockerExport(container string) (io.ReadCloser, error) { log.Debugf("docker export: %s", container) cli, err := dockerClient() if err != nil { - return []byte{}, errors.New("could not initialize Docker API client") + return nil, errors.New("could not initialize Docker API client") } responseBody, err := cli.ContainerExport(context.Background(), container) if err != nil { - return []byte{}, err - } - defer responseBody.Close() - - output := bytes.NewBuffer(nil) - _, err = io.Copy(output, responseBody) - if err != nil { - return []byte{}, err + return nil, err } - return output.Bytes(), nil + return responseBody, err } func dockerRm(container string) error { diff --git a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/image.go b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/image.go index 9f2a3504e..22def6cfc 100644 --- a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/image.go +++ b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/image.go @@ -82,7 +82,7 @@ func tarPrefix(path string, tw tarWriter) error { } // ImageTar takes a Docker image and outputs it to a tar stream -func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) error { +func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull bool, resolv string) (e error) { log.Debugf("image tar: %s %s", ref, prefix) if prefix != "" && prefix[len(prefix)-1] != byte('/') { return fmt.Errorf("prefix does not end with /: %s", prefix) @@ -119,15 +119,17 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull if err != nil { return fmt.Errorf("Failed to docker export container from container %s: %v", container, err) } - err = dockerRm(container) - if err != nil { - return fmt.Errorf("Failed to docker rm container %s: %v", container, err) - } + defer func() { + contents.Close() + + if err := dockerRm(container); e == nil && err != nil { + e = fmt.Errorf("Failed to docker rm container %s: %v", container, err) + } + }() // now we need to filter out some files from the resulting tar archive - r := bytes.NewReader(contents) - tr := tar.NewReader(r) + tr := tar.NewReader(contents) for { hdr, err := tr.Next() diff --git a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/linuxkit.go b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/linuxkit.go index 9c142bc39..67449e372 100644 --- a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/linuxkit.go +++ b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/linuxkit.go @@ -1,7 +1,6 @@ package moby import ( - "bytes" "crypto/sha256" "fmt" "io" @@ -18,13 +17,13 @@ kernel: image: linuxkit/kernel:4.9.39 cmdline: "console=ttyS0" init: - - linuxkit/init:9250948d0de494df8a811edb3242b4584057cfe4 - - linuxkit/runc:abc3f292653e64a2fd488e9675ace19a55ec7023 + - linuxkit/init:00ab58c9681a0bf42b2e35134c1ccf1591ebb64d + - linuxkit/runc:f5960b83a8766ae083efc744fa63dbf877450e4f onboot: - name: mkimage - image: linuxkit/mkimage:e439d6108466186948ca7ea2a293fc6c1d1183fa + image: linuxkit/mkimage:a63b8ee4c5de335afc32ba850e0af319b25b96c0 - name: poweroff - image: linuxkit/poweroff:bccfe1cb04fc7bb9f03613d2314f38abd2620f29 + image: linuxkit/poweroff:3845c4d64d47a1ea367806be5547e44594b0fa91 trust: org: - linuxkit @@ -58,9 +57,21 @@ func ensureLinuxkitImage(name string) error { return err } // TODO pass through --pull to here - buf := new(bytes.Buffer) - Build(m, buf, false, "") - image := buf.Bytes() + tf, err := ioutil.TempFile("", "") + if err != nil { + return err + } + defer os.Remove(tf.Name()) + Build(m, tf, false, "") + if err := tf.Close(); err != nil { + return err + } + + image, err := os.Open(tf.Name()) + if err != nil { + return err + } + defer image.Close() kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) diff --git a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/output.go b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/output.go index bc83afe73..5cd649104 100644 --- a/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/output.go +++ b/src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/output.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "fmt" + "io" "io/ioutil" "os" "runtime" @@ -24,8 +25,8 @@ const ( rpi3 = "linuxkit/mkimage-rpi3:0735656fff247ca978135e3aeb62864adc612180" ) -var outFuns = map[string]func(string, []byte, int) error{ - "kernel+initrd": func(base string, image []byte, size int) error { +var outFuns = map[string]func(string, io.Reader, int) error{ + "kernel+initrd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -36,7 +37,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "tar-kernel-initrd": func(base string, image []byte, size int) error { + "tar-kernel-initrd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -46,21 +47,21 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "iso-bios": func(base string, image []byte, size int) error { + "iso-bios": func(base string, image io.Reader, size int) error { err := outputIso(isoBios, base+".iso", image) if err != nil { return fmt.Errorf("Error writing iso-bios output: %v", err) } return nil }, - "iso-efi": func(base string, image []byte, size int) error { + "iso-efi": func(base string, image io.Reader, size int) error { err := outputIso(isoEfi, base+"-efi.iso", image) if err != nil { return fmt.Errorf("Error writing iso-efi output: %v", err) } return nil }, - "raw-bios": func(base string, image []byte, size int) error { + "raw-bios": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -71,7 +72,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "raw-efi": func(base string, image []byte, size int) error { + "raw-efi": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -82,7 +83,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "aws": func(base string, image []byte, size int) error { + "aws": func(base string, image io.Reader, size int) error { filename := base + ".raw" log.Infof(" %s", filename) kernel, initrd, cmdline, err := tarToInitrd(image) @@ -95,7 +96,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "gcp": func(base string, image []byte, size int) error { + "gcp": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -106,7 +107,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "qcow2-bios": func(base string, image []byte, size int) error { + "qcow2-bios": func(base string, image io.Reader, size int) error { filename := base + ".qcow2" log.Infof(" %s", filename) kernel, initrd, cmdline, err := tarToInitrd(image) @@ -119,7 +120,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "vhd": func(base string, image []byte, size int) error { + "vhd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -130,7 +131,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "dynamic-vhd": func(base string, image []byte, size int) error { + "dynamic-vhd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -141,7 +142,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "vmdk": func(base string, image []byte, size int) error { + "vmdk": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -152,7 +153,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "rpi3": func(base string, image []byte, size int) error { + "rpi3": func(base string, image io.Reader, size int) error { if runtime.GOARCH != "arm64" { return fmt.Errorf("Raspberry Pi output currently only supported on arm64") } @@ -197,7 +198,7 @@ func ValidateFormats(formats []string) error { } // Formats generates all the specified output formats -func Formats(base string, image []byte, formats []string, size int) error { +func Formats(base string, image string, formats []string, size int) error { log.Debugf("format: %v %s", formats, base) err := ValidateFormats(formats) @@ -205,20 +206,23 @@ func Formats(base string, image []byte, formats []string, size int) error { return err } for _, o := range formats { - f := outFuns[o] - err := f(base, image, size) + ir, err := os.Open(image) if err != nil { return err } + defer ir.Close() + f := outFuns[o] + if err := f(base, ir, size); err != nil { + return err + } } return nil } -func tarToInitrd(image []byte) ([]byte, []byte, string, error) { +func tarToInitrd(r io.Reader) ([]byte, []byte, string, error) { w := new(bytes.Buffer) iw := initrd.NewWriter(w) - r := bytes.NewReader(image) tr := tar.NewReader(r) kernel, cmdline, err := initrd.CopySplitTar(iw, tr) if err != nil { @@ -288,7 +292,7 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str return dockerRun(buf, output, true, image, cmdline) } -func outputIso(image, filename string, filesystem []byte) error { +func outputIso(image, filename string, filesystem io.Reader) error { log.Debugf("output ISO: %s %s", image, filename) log.Infof(" %s", filename) output, err := os.Create(filename) @@ -296,10 +300,10 @@ func outputIso(image, filename string, filesystem []byte) error { return err } defer output.Close() - return dockerRun(bytes.NewBuffer(filesystem), output, true, image) + return dockerRun(filesystem, output, true, image) } -func outputRPi3(image, filename string, filesystem []byte) error { +func outputRPi3(image, filename string, filesystem io.Reader) error { log.Debugf("output RPi3: %s %s", image, filename) log.Infof(" %s", filename) output, err := os.Create(filename) @@ -307,7 +311,7 @@ func outputRPi3(image, filename string, filesystem []byte) error { return err } defer output.Close() - return dockerRun(bytes.NewBuffer(filesystem), output, true, image) + return dockerRun(filesystem, output, true, image) } func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline string) error {