diff --git a/cmd/moby/build.go b/cmd/moby/build.go index dd67b8049..ef8f623e0 100644 --- a/cmd/moby/build.go +++ b/cmd/moby/build.go @@ -185,14 +185,18 @@ func build(args []string) { m.Trust = moby.TrustConfig{} } - var buf *bytes.Buffer + var tf *os.File var w io.Writer if outputFile != nil { w = outputFile } else { - buf = new(bytes.Buffer) - w = buf + if tf, err = ioutil.TempFile("", ""); err != nil { + log.Fatalf("Error creating tempfile: %v", err) + } + defer os.Remove(tf.Name()) + w = tf } + // this is a weird interface, but currently only streamable types can have additional files // need to split up the base tarball outputs from the secondary stages var tp string @@ -205,7 +209,11 @@ func build(args []string) { } if outputFile == nil { - image := buf.Bytes() + image := tf.Name() + if err := tf.Close(); err != nil { + log.Fatalf("Error closing tempfile: %v", err) + } + log.Infof("Create outputs:") err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size) if err != nil { diff --git a/cmd/moby/main.go b/cmd/moby/main.go index 8fa32e4a1..55cdde754 100644 --- a/cmd/moby/main.go +++ b/cmd/moby/main.go @@ -5,6 +5,8 @@ import ( "fmt" "os" "path/filepath" + "runtime" + "runtime/pprof" "github.com/moby/tool/src/moby" log "github.com/sirupsen/logrus" @@ -60,6 +62,8 @@ func main() { } flagQuiet := flag.Bool("q", false, "Quiet execution") flagVerbose := flag.Bool("v", false, "Verbose execution") + flagCPUProfile := flag.String("cpuprofile", "", "write cpu profile to `file`") + flagMemProfile := flag.String("memprofile", "", "write mem profile to `file`") // config and cache directory flagConfigDir := flag.String("config", defaultMobyConfigDir(), "Configuration directory") @@ -101,6 +105,17 @@ func main() { } moby.MobyDir = mobyDir + if *flagCPUProfile != "" { + f, err := os.Create(*flagCPUProfile) + if err != nil { + log.Fatal("could not create CPU profile: ", err) + } + if err := pprof.StartCPUProfile(f); err != nil { + log.Fatal("could not start CPU profile: ", err) + } + defer pprof.StopCPUProfile() + } + switch args[0] { case "build": build(args[1:]) @@ -113,4 +128,16 @@ func main() { flag.Usage() os.Exit(1) } + + if *flagMemProfile != "" { + f, err := os.Create(*flagMemProfile) + if err != nil { + log.Fatal("could not create memory profile: ", err) + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.Fatal("could not write memory profile: ", err) + } + f.Close() + } } diff --git a/src/moby/docker.go b/src/moby/docker.go index 640cf8119..296a61108 100644 --- a/src/moby/docker.go +++ b/src/moby/docker.go @@ -4,7 +4,6 @@ package moby // and also using the Docker API not shelling out import ( - "bytes" "errors" "fmt" "io" @@ -81,25 +80,18 @@ func dockerCreate(image string) (string, error) { return respBody.ID, nil } -func dockerExport(container string) ([]byte, error) { +func dockerExport(container string) (io.ReadCloser, error) { log.Debugf("docker export: %s", container) cli, err := dockerClient() if err != nil { - return []byte{}, errors.New("could not initialize Docker API client") + return nil, errors.New("could not initialize Docker API client") } responseBody, err := cli.ContainerExport(context.Background(), container) if err != nil { - return []byte{}, err - } - defer responseBody.Close() - - output := bytes.NewBuffer(nil) - _, err = io.Copy(output, responseBody) - if err != nil { - return []byte{}, err + return nil, err } - return output.Bytes(), nil + return responseBody, err } func dockerRm(container string) error { diff --git a/src/moby/image.go b/src/moby/image.go index 9f2a3504e..3b3426ebe 100644 --- a/src/moby/image.go +++ b/src/moby/image.go @@ -119,6 +119,8 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull if err != nil { return fmt.Errorf("Failed to docker export container from container %s: %v", container, err) } + defer contents.Close() + err = dockerRm(container) if err != nil { return fmt.Errorf("Failed to docker rm container %s: %v", container, err) @@ -126,8 +128,7 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull // now we need to filter out some files from the resulting tar archive - r := bytes.NewReader(contents) - tr := tar.NewReader(r) + tr := tar.NewReader(contents) for { hdr, err := tr.Next() diff --git a/src/moby/linuxkit.go b/src/moby/linuxkit.go index 3317f1ed6..67449e372 100644 --- a/src/moby/linuxkit.go +++ b/src/moby/linuxkit.go @@ -1,7 +1,6 @@ package moby import ( - "bytes" "crypto/sha256" "fmt" "io" @@ -58,9 +57,21 @@ func ensureLinuxkitImage(name string) error { return err } // TODO pass through --pull to here - buf := new(bytes.Buffer) - Build(m, buf, false, "") - image := buf.Bytes() + tf, err := ioutil.TempFile("", "") + if err != nil { + return err + } + defer os.Remove(tf.Name()) + Build(m, tf, false, "") + if err := tf.Close(); err != nil { + return err + } + + image, err := os.Open(tf.Name()) + if err != nil { + return err + } + defer image.Close() kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) diff --git a/src/moby/output.go b/src/moby/output.go index bc83afe73..5cd649104 100644 --- a/src/moby/output.go +++ b/src/moby/output.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "fmt" + "io" "io/ioutil" "os" "runtime" @@ -24,8 +25,8 @@ const ( rpi3 = "linuxkit/mkimage-rpi3:0735656fff247ca978135e3aeb62864adc612180" ) -var outFuns = map[string]func(string, []byte, int) error{ - "kernel+initrd": func(base string, image []byte, size int) error { +var outFuns = map[string]func(string, io.Reader, int) error{ + "kernel+initrd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -36,7 +37,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "tar-kernel-initrd": func(base string, image []byte, size int) error { + "tar-kernel-initrd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -46,21 +47,21 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "iso-bios": func(base string, image []byte, size int) error { + "iso-bios": func(base string, image io.Reader, size int) error { err := outputIso(isoBios, base+".iso", image) if err != nil { return fmt.Errorf("Error writing iso-bios output: %v", err) } return nil }, - "iso-efi": func(base string, image []byte, size int) error { + "iso-efi": func(base string, image io.Reader, size int) error { err := outputIso(isoEfi, base+"-efi.iso", image) if err != nil { return fmt.Errorf("Error writing iso-efi output: %v", err) } return nil }, - "raw-bios": func(base string, image []byte, size int) error { + "raw-bios": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -71,7 +72,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "raw-efi": func(base string, image []byte, size int) error { + "raw-efi": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -82,7 +83,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "aws": func(base string, image []byte, size int) error { + "aws": func(base string, image io.Reader, size int) error { filename := base + ".raw" log.Infof(" %s", filename) kernel, initrd, cmdline, err := tarToInitrd(image) @@ -95,7 +96,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "gcp": func(base string, image []byte, size int) error { + "gcp": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -106,7 +107,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "qcow2-bios": func(base string, image []byte, size int) error { + "qcow2-bios": func(base string, image io.Reader, size int) error { filename := base + ".qcow2" log.Infof(" %s", filename) kernel, initrd, cmdline, err := tarToInitrd(image) @@ -119,7 +120,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "vhd": func(base string, image []byte, size int) error { + "vhd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -130,7 +131,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "dynamic-vhd": func(base string, image []byte, size int) error { + "dynamic-vhd": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -141,7 +142,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "vmdk": func(base string, image []byte, size int) error { + "vmdk": func(base string, image io.Reader, size int) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) @@ -152,7 +153,7 @@ var outFuns = map[string]func(string, []byte, int) error{ } return nil }, - "rpi3": func(base string, image []byte, size int) error { + "rpi3": func(base string, image io.Reader, size int) error { if runtime.GOARCH != "arm64" { return fmt.Errorf("Raspberry Pi output currently only supported on arm64") } @@ -197,7 +198,7 @@ func ValidateFormats(formats []string) error { } // Formats generates all the specified output formats -func Formats(base string, image []byte, formats []string, size int) error { +func Formats(base string, image string, formats []string, size int) error { log.Debugf("format: %v %s", formats, base) err := ValidateFormats(formats) @@ -205,20 +206,23 @@ func Formats(base string, image []byte, formats []string, size int) error { return err } for _, o := range formats { - f := outFuns[o] - err := f(base, image, size) + ir, err := os.Open(image) if err != nil { return err } + defer ir.Close() + f := outFuns[o] + if err := f(base, ir, size); err != nil { + return err + } } return nil } -func tarToInitrd(image []byte) ([]byte, []byte, string, error) { +func tarToInitrd(r io.Reader) ([]byte, []byte, string, error) { w := new(bytes.Buffer) iw := initrd.NewWriter(w) - r := bytes.NewReader(image) tr := tar.NewReader(r) kernel, cmdline, err := initrd.CopySplitTar(iw, tr) if err != nil { @@ -288,7 +292,7 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str return dockerRun(buf, output, true, image, cmdline) } -func outputIso(image, filename string, filesystem []byte) error { +func outputIso(image, filename string, filesystem io.Reader) error { log.Debugf("output ISO: %s %s", image, filename) log.Infof(" %s", filename) output, err := os.Create(filename) @@ -296,10 +300,10 @@ func outputIso(image, filename string, filesystem []byte) error { return err } defer output.Close() - return dockerRun(bytes.NewBuffer(filesystem), output, true, image) + return dockerRun(filesystem, output, true, image) } -func outputRPi3(image, filename string, filesystem []byte) error { +func outputRPi3(image, filename string, filesystem io.Reader) error { log.Debugf("output RPi3: %s %s", image, filename) log.Infof(" %s", filename) output, err := os.Create(filename) @@ -307,7 +311,7 @@ func outputRPi3(image, filename string, filesystem []byte) error { return err } defer output.Close() - return dockerRun(bytes.NewBuffer(filesystem), output, true, image) + return dockerRun(filesystem, output, true, image) } func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline string) error {