From 9558740c11285224f01f2f6e05d522e75b85718c Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 6 Dec 2017 15:54:18 +0000 Subject: [PATCH 1/3] Add cpu and mem profiling options Following https://golang.org/pkg/runtime/pprof/. When attempting to build images in https://github.com/linuxkit/kubernetes CI the process is mysteriously being SIGKILL'd, which I think might be down to OOMing due to the resource limits placed on the build container. I haven't done so yet but I'm intending to use these options to investigate and they seem potentially useful in any case, even if this turns out to be a red-herring. Signed-off-by: Ian Campbell --- cmd/moby/main.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/cmd/moby/main.go b/cmd/moby/main.go index 8fa32e4a1..55cdde754 100644 --- a/cmd/moby/main.go +++ b/cmd/moby/main.go @@ -5,6 +5,8 @@ import ( "fmt" "os" "path/filepath" + "runtime" + "runtime/pprof" "github.com/moby/tool/src/moby" log "github.com/sirupsen/logrus" @@ -60,6 +62,8 @@ func main() { } flagQuiet := flag.Bool("q", false, "Quiet execution") flagVerbose := flag.Bool("v", false, "Verbose execution") + flagCPUProfile := flag.String("cpuprofile", "", "write cpu profile to `file`") + flagMemProfile := flag.String("memprofile", "", "write mem profile to `file`") // config and cache directory flagConfigDir := flag.String("config", defaultMobyConfigDir(), "Configuration directory") @@ -101,6 +105,17 @@ func main() { } moby.MobyDir = mobyDir + if *flagCPUProfile != "" { + f, err := os.Create(*flagCPUProfile) + if err != nil { + log.Fatal("could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + } + switch args[0] { case "build": build(args[1:]) @@ -113,4 +128,16 @@ func main() { flag.Usage() os.Exit(1) } + + if *flagMemProfile != "" { + f, err := os.Create(*flagMemProfile) + if err != nil { + log.Fatal("could not create memory profile: ", err) + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.Fatal("could not write memory profile: ", err) + } + f.Close() + } } From 9f44acf8e384376eaa46cd5355360ed521d718dd Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 6 Dec 2017 14:53:18 +0000 Subject: [PATCH 2/3] Generate intermediate image into a temp file All of the `output*` functions took a `[]byte` and immediately wrapped it in a `bytes.Buffer` to produce an `io.Reader`. Make them take an `io.Reader` instead and satisfy this further up the call chain by directing `moby.Build` to output to a temp file instead of another `bytes.Buffer`. In my test case (building kube master image) this reduces Maximum RSS (as measured by time(1)) from 6.7G to 2.8G and overall allocations from 9.7G to 5.3G. When building a tar (output to /dev/null) the Maximum RSS fell slightly from 2.2G to 2.1G. Overall allocations remained stable at around 5.3G. Signed-off-by: Ian Campbell --- cmd/moby/build.go | 16 ++++++++++---- src/moby/linuxkit.go | 19 +++++++++++++---- src/moby/output.go | 50 ++++++++++++++++++++++++-------------------- 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/cmd/moby/build.go b/cmd/moby/build.go index dd67b8049..ef8f623e0 100644 --- a/cmd/moby/build.go +++ b/cmd/moby/build.go @@ -185,14 +185,18 @@ func build(args []string) { m.Trust = moby.TrustConfig{} } - var buf *bytes.Buffer + var tf *os.File var w io.Writer if outputFile != nil { w = outputFile } else { - buf = new(bytes.Buffer) - w = buf + if tf, err = ioutil.TempFile("", ""); err != nil { + log.Fatalf("Error creating tempfile: %v", err) + } + defer os.Remove(tf.Name()) + w = tf } + // this is a weird interface, but currently only streamable types can have additional files // need to split up the base tarball outputs from the secondary stages var tp string @@ -205,7 +209,11 @@ func build(args []string) { } if outputFile == nil { - image := buf.Bytes() + image := tf.Name() + if err := tf.Close(); err != nil { + log.Fatalf("Error closing tempfile: %v", err) + } + log.Infof("Create outputs:") err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size) if err != nil { diff --git a/src/moby/linuxkit.go b/src/moby/linuxkit.go index 3317f1ed6..67449e372 100644 --- a/src/moby/linuxkit.go +++ b/src/moby/linuxkit.go @@ -1,7 +1,6 @@ package moby import ( - "bytes" "crypto/sha256" "fmt" "io" @@ -58,9 +57,21 @@ func ensureLinuxkitImage(name string) error { return err } // TODO pass through --pull to here - buf := new(bytes.Buffer) - Build(m, buf, false, "") - image := buf.Bytes() + tf, err := ioutil.TempFile("", "") + if err != nil { + return err + } + defer os.Remove(tf.Name()) + Build(m, tf, false, "") + if err := tf.Close(); err != nil { + return err + } + + image, err := os.Open(tf.Name()) + if err != nil { + return err + } + defer image.Close() kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) diff --git a/src/moby/output.go b/src/moby/output.go index bc83afe73..5cd649104 100644 --- a/src/moby/output.go +++ b/src/moby/output.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "fmt" + "io" "io/ioutil" "os" "runtime" @@ -24,8 +25,8 @@ const ( rpi3 = "linuxkit/mkimage-rpi3:0735656fff247ca978135e3aeb62864adc612180" ) -var outFuns = map[string]func(string, []byte, int) error{ - "kernel+initrd": func(base string, image []byte, size int) error { +var outFuns = map[string]func(string, io.Reader, int) error{ + "kernel+initrd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -36,7 +37,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "tar-kernel-initrd": func(base string, image []byte, size int) error { + "tar-kernel-initrd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -46,21 +47,21 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "iso-bios": func(base string, image []byte, size int) error { + "iso-bios": func(base string, image io.Reader, size int) error { err := outputIso(isoBios, base+".iso", image) if err != nil { return fmt.Errorf("Error writing iso-bios output: %v", err) } return nil }, - "iso-efi": func(base string, image []byte, size int) error { + "iso-efi": func(base string, image io.Reader, size int) error { err := outputIso(isoEfi, base+"-efi.iso", image) if err != nil { return fmt.Errorf("Error writing iso-efi output: %v", err) } return nil }, - "raw-bios": func(base string, image []byte, size int) error { + "raw-bios": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -71,7 +72,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "raw-efi": func(base string, image []byte, size int) error { + "raw-efi": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -82,7 +83,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "aws": func(base string, image []byte, size int) error { + "aws": func(base string, image io.Reader, size int) error { filename := base + ".raw" log.Infof(" %s", filename) kernel, initrd, cmdline, err := tarToInitrd(image) @@ -95,7 +96,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "gcp": func(base string, image []byte, size int) error { + "gcp": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -106,7 +107,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "qcow2-bios": func(base string, image []byte, size int) error { + "qcow2-bios": func(base string, image io.Reader, size int) error { filename := base + ".qcow2" log.Infof(" %s", filename) kernel, initrd, cmdline, err := tarToInitrd(image) @@ -119,7 +120,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "vhd": func(base string, image []byte, size int) error { + "vhd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -130,7 +131,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "dynamic-vhd": func(base string, image []byte, size int) error { + "dynamic-vhd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -141,7 +142,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "vmdk": func(base string, image []byte, size int) error { + "vmdk": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -152,7 +153,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "rpi3": func(base string, image []byte, size int) error { + "rpi3": func(base string, image io.Reader, size int) error { if runtime.GOARCH != "arm64" { return fmt.Errorf("Raspberry Pi output currently only supported on arm64") } @@ -197,7 +198,7 @@ func ValidateFormats(formats []string) error { } // Formats generates all the specified output formats -func Formats(base string, image []byte, formats []string, size int) error { +func Formats(base string, image string, formats []string, size int) error { log.Debugf("format: %v %s", formats, base) err := ValidateFormats(formats) @@ -205,20 +206,23 @@ func Formats(base string, image []byte, formats []string, size int) error { return err } for _, o := range formats { - f := outFuns[o] - err := f(base, image, size) + ir, err := os.Open(image) if err != nil { return err } + defer ir.Close() + f := outFuns[o] + if err := f(base, ir, size); err != nil { + return err + } } return nil } -func tarToInitrd(image []byte) ([]byte, []byte, string, error) { +func tarToInitrd(r io.Reader) ([]byte, []byte, string, error) { w := new(bytes.Buffer) iw := initrd.NewWriter(w) - r := bytes.NewReader(image) tr := tar.NewReader(r) kernel, cmdline, err := initrd.CopySplitTar(iw, tr) if err != nil { @@ -288,7 +292,7 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str return dockerRun(buf, output, true, image, cmdline) } -func outputIso(image, filename string, filesystem []byte) error { +func outputIso(image, filename string, filesystem io.Reader) error { log.Debugf("output ISO: %s %s", image, filename) log.Infof(" %s", filename) output, err := os.Create(filename) @@ -296,10 +300,10 @@ func outputIso(image, filename string, filesystem []byte) error { return err } defer output.Close() - return dockerRun(bytes.NewBuffer(filesystem), output, true, image) + return dockerRun(filesystem, output, true, image) } -func outputRPi3(image, filename string, filesystem []byte) error { +func outputRPi3(image, filename string, filesystem io.Reader) error { log.Debugf("output RPi3: %s %s", image, filename) log.Infof(" %s", filename) output, err := os.Create(filename) @@ -307,7 +311,7 @@ func outputRPi3(image, filename string, filesystem []byte) error { return err } defer output.Close() - return dockerRun(bytes.NewBuffer(filesystem), output, true, image) + return dockerRun(filesystem, output, true, image) } func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline string) error { From 3045a80c851f0c1ade20f925f67520e47e90bb0a Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 6 Dec 2017 15:05:04 +0000 Subject: [PATCH 3/3] Stream `docker export` directly to consumer Rather than queueing up into a `bytes.Buffer`. In my test case (building kube master image) this reduces Maximum RSS (as measured by time(1)) compared with the previous patch from 2.8G to 110M. The tar output case goes from 2.1G to 110M also. Overall allocations are ~715M in both cases. Signed-off-by: Ian Campbell --- src/moby/docker.go | 16 ++++------------ src/moby/image.go | 5 +++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/src/moby/docker.go b/src/moby/docker.go index 640cf8119..296a61108 100644 --- a/src/moby/docker.go +++ b/src/moby/docker.go @@ -4,7 +4,6 @@ package moby // and also using the Docker API not shelling out import ( - "bytes" "errors" "fmt" "io" @@ -81,25 +80,18 @@ func dockerCreate(image string) (string, error) { return respBody.ID, nil } -func dockerExport(container string) ([]byte, error) { +func dockerExport(container string) (io.ReadCloser, error) { log.Debugf("docker export: %s", container) cli, err := dockerClient() if err != nil { - return []byte{}, errors.New("could not initialize Docker API client") + return nil, errors.New("could not initialize Docker API client") } responseBody, err := cli.ContainerExport(context.Background(), container) if err != nil { - return []byte{}, err - } - defer responseBody.Close() - - output := bytes.NewBuffer(nil) - _, err = io.Copy(output, responseBody) - if err != nil { - return []byte{}, err + return nil, err } - return output.Bytes(), nil + return responseBody, err } func dockerRm(container string) error { diff --git a/src/moby/image.go b/src/moby/image.go index 9f2a3504e..3b3426ebe 100644 --- a/src/moby/image.go +++ b/src/moby/image.go @@ -119,6 +119,8 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull if err != nil { return fmt.Errorf("Failed to docker export container from container %s: %v", container, err) } + defer contents.Close() + err = dockerRm(container) if err != nil { return fmt.Errorf("Failed to docker rm container %s: %v", container, err) @@ -126,8 +128,7 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull // now we need to filter out some files from the resulting tar archive - r := bytes.NewReader(contents) - tr := tar.NewReader(r) + tr := tar.NewReader(contents) for { hdr, err := tr.Next()