mirror of
https://github.com/linuxkit/linuxkit.git
synced 2025-07-23 02:51:55 +00:00
Merge pull request #191 from ijc/reduce-memory-via-tempfiles
Reduce maximum memory usage via tempfiles
This commit is contained in:
commit
ebd7228a44
@ -185,14 +185,18 @@ func build(args []string) {
|
||||
m.Trust = moby.TrustConfig{}
|
||||
}
|
||||
|
||||
var buf *bytes.Buffer
|
||||
var tf *os.File
|
||||
var w io.Writer
|
||||
if outputFile != nil {
|
||||
w = outputFile
|
||||
} else {
|
||||
buf = new(bytes.Buffer)
|
||||
w = buf
|
||||
if tf, err = ioutil.TempFile("", ""); err != nil {
|
||||
log.Fatalf("Error creating tempfile: %v", err)
|
||||
}
|
||||
defer os.Remove(tf.Name())
|
||||
w = tf
|
||||
}
|
||||
|
||||
// this is a weird interface, but currently only streamable types can have additional files
|
||||
// need to split up the base tarball outputs from the secondary stages
|
||||
var tp string
|
||||
@ -205,7 +209,11 @@ func build(args []string) {
|
||||
}
|
||||
|
||||
if outputFile == nil {
|
||||
image := buf.Bytes()
|
||||
image := tf.Name()
|
||||
if err := tf.Close(); err != nil {
|
||||
log.Fatalf("Error closing tempfile: %v", err)
|
||||
}
|
||||
|
||||
log.Infof("Create outputs:")
|
||||
err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size)
|
||||
if err != nil {
|
||||
|
@ -5,6 +5,8 @@ import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"runtime/pprof"
|
||||
|
||||
"github.com/moby/tool/src/moby"
|
||||
log "github.com/sirupsen/logrus"
|
||||
@ -60,6 +62,8 @@ func main() {
|
||||
}
|
||||
flagQuiet := flag.Bool("q", false, "Quiet execution")
|
||||
flagVerbose := flag.Bool("v", false, "Verbose execution")
|
||||
flagCPUProfile := flag.String("cpuprofile", "", "write cpu profile to `file`")
|
||||
flagMemProfile := flag.String("memprofile", "", "write mem profile to `file`")
|
||||
|
||||
// config and cache directory
|
||||
flagConfigDir := flag.String("config", defaultMobyConfigDir(), "Configuration directory")
|
||||
@ -101,6 +105,17 @@ func main() {
|
||||
}
|
||||
moby.MobyDir = mobyDir
|
||||
|
||||
if *flagCPUProfile != "" {
|
||||
f, err := os.Create(*flagCPUProfile)
|
||||
if err != nil {
|
||||
log.Fatal("could not create CPU profile: ", err)
|
||||
}
|
||||
if err := pprof.StartCPUProfile(f); err != nil {
|
||||
log.Fatal("could not start CPU profile: ", err)
|
||||
}
|
||||
defer pprof.StopCPUProfile()
|
||||
}
|
||||
|
||||
switch args[0] {
|
||||
case "build":
|
||||
build(args[1:])
|
||||
@ -113,4 +128,16 @@ func main() {
|
||||
flag.Usage()
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
if *flagMemProfile != "" {
|
||||
f, err := os.Create(*flagMemProfile)
|
||||
if err != nil {
|
||||
log.Fatal("could not create memory profile: ", err)
|
||||
}
|
||||
runtime.GC() // get up-to-date statistics
|
||||
if err := pprof.WriteHeapProfile(f); err != nil {
|
||||
log.Fatal("could not write memory profile: ", err)
|
||||
}
|
||||
f.Close()
|
||||
}
|
||||
}
|
||||
|
@ -4,7 +4,6 @@ package moby
|
||||
// and also using the Docker API not shelling out
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -81,25 +80,18 @@ func dockerCreate(image string) (string, error) {
|
||||
return respBody.ID, nil
|
||||
}
|
||||
|
||||
func dockerExport(container string) ([]byte, error) {
|
||||
func dockerExport(container string) (io.ReadCloser, error) {
|
||||
log.Debugf("docker export: %s", container)
|
||||
cli, err := dockerClient()
|
||||
if err != nil {
|
||||
return []byte{}, errors.New("could not initialize Docker API client")
|
||||
return nil, errors.New("could not initialize Docker API client")
|
||||
}
|
||||
responseBody, err := cli.ContainerExport(context.Background(), container)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
}
|
||||
defer responseBody.Close()
|
||||
|
||||
output := bytes.NewBuffer(nil)
|
||||
_, err = io.Copy(output, responseBody)
|
||||
if err != nil {
|
||||
return []byte{}, err
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return output.Bytes(), nil
|
||||
return responseBody, err
|
||||
}
|
||||
|
||||
func dockerRm(container string) error {
|
||||
|
@ -119,6 +119,8 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to docker export container from container %s: %v", container, err)
|
||||
}
|
||||
defer contents.Close()
|
||||
|
||||
err = dockerRm(container)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Failed to docker rm container %s: %v", container, err)
|
||||
@ -126,8 +128,7 @@ func ImageTar(ref *reference.Spec, prefix string, tw tarWriter, trust bool, pull
|
||||
|
||||
// now we need to filter out some files from the resulting tar archive
|
||||
|
||||
r := bytes.NewReader(contents)
|
||||
tr := tar.NewReader(r)
|
||||
tr := tar.NewReader(contents)
|
||||
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
|
@ -1,7 +1,6 @@
|
||||
package moby
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -58,9 +57,21 @@ func ensureLinuxkitImage(name string) error {
|
||||
return err
|
||||
}
|
||||
// TODO pass through --pull to here
|
||||
buf := new(bytes.Buffer)
|
||||
Build(m, buf, false, "")
|
||||
image := buf.Bytes()
|
||||
tf, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tf.Name())
|
||||
Build(m, tf, false, "")
|
||||
if err := tf.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
image, err := os.Open(tf.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer image.Close()
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
|
@ -4,6 +4,7 @@ import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"runtime"
|
||||
@ -24,8 +25,8 @@ const (
|
||||
rpi3 = "linuxkit/mkimage-rpi3:0735656fff247ca978135e3aeb62864adc612180"
|
||||
)
|
||||
|
||||
var outFuns = map[string]func(string, []byte, int) error{
|
||||
"kernel+initrd": func(base string, image []byte, size int) error {
|
||||
var outFuns = map[string]func(string, io.Reader, int) error{
|
||||
"kernel+initrd": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -36,7 +37,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"tar-kernel-initrd": func(base string, image []byte, size int) error {
|
||||
"tar-kernel-initrd": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -46,21 +47,21 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"iso-bios": func(base string, image []byte, size int) error {
|
||||
"iso-bios": func(base string, image io.Reader, size int) error {
|
||||
err := outputIso(isoBios, base+".iso", image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error writing iso-bios output: %v", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"iso-efi": func(base string, image []byte, size int) error {
|
||||
"iso-efi": func(base string, image io.Reader, size int) error {
|
||||
err := outputIso(isoEfi, base+"-efi.iso", image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error writing iso-efi output: %v", err)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"raw-bios": func(base string, image []byte, size int) error {
|
||||
"raw-bios": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -71,7 +72,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"raw-efi": func(base string, image []byte, size int) error {
|
||||
"raw-efi": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -82,7 +83,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"aws": func(base string, image []byte, size int) error {
|
||||
"aws": func(base string, image io.Reader, size int) error {
|
||||
filename := base + ".raw"
|
||||
log.Infof(" %s", filename)
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
@ -95,7 +96,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"gcp": func(base string, image []byte, size int) error {
|
||||
"gcp": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -106,7 +107,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"qcow2-bios": func(base string, image []byte, size int) error {
|
||||
"qcow2-bios": func(base string, image io.Reader, size int) error {
|
||||
filename := base + ".qcow2"
|
||||
log.Infof(" %s", filename)
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
@ -119,7 +120,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"vhd": func(base string, image []byte, size int) error {
|
||||
"vhd": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -130,7 +131,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"dynamic-vhd": func(base string, image []byte, size int) error {
|
||||
"dynamic-vhd": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -141,7 +142,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"vmdk": func(base string, image []byte, size int) error {
|
||||
"vmdk": func(base string, image io.Reader, size int) error {
|
||||
kernel, initrd, cmdline, err := tarToInitrd(image)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error converting to initrd: %v", err)
|
||||
@ -152,7 +153,7 @@ var outFuns = map[string]func(string, []byte, int) error{
|
||||
}
|
||||
return nil
|
||||
},
|
||||
"rpi3": func(base string, image []byte, size int) error {
|
||||
"rpi3": func(base string, image io.Reader, size int) error {
|
||||
if runtime.GOARCH != "arm64" {
|
||||
return fmt.Errorf("Raspberry Pi output currently only supported on arm64")
|
||||
}
|
||||
@ -197,7 +198,7 @@ func ValidateFormats(formats []string) error {
|
||||
}
|
||||
|
||||
// Formats generates all the specified output formats
|
||||
func Formats(base string, image []byte, formats []string, size int) error {
|
||||
func Formats(base string, image string, formats []string, size int) error {
|
||||
log.Debugf("format: %v %s", formats, base)
|
||||
|
||||
err := ValidateFormats(formats)
|
||||
@ -205,20 +206,23 @@ func Formats(base string, image []byte, formats []string, size int) error {
|
||||
return err
|
||||
}
|
||||
for _, o := range formats {
|
||||
f := outFuns[o]
|
||||
err := f(base, image, size)
|
||||
ir, err := os.Open(image)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer ir.Close()
|
||||
f := outFuns[o]
|
||||
if err := f(base, ir, size); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func tarToInitrd(image []byte) ([]byte, []byte, string, error) {
|
||||
func tarToInitrd(r io.Reader) ([]byte, []byte, string, error) {
|
||||
w := new(bytes.Buffer)
|
||||
iw := initrd.NewWriter(w)
|
||||
r := bytes.NewReader(image)
|
||||
tr := tar.NewReader(r)
|
||||
kernel, cmdline, err := initrd.CopySplitTar(iw, tr)
|
||||
if err != nil {
|
||||
@ -288,7 +292,7 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str
|
||||
return dockerRun(buf, output, true, image, cmdline)
|
||||
}
|
||||
|
||||
func outputIso(image, filename string, filesystem []byte) error {
|
||||
func outputIso(image, filename string, filesystem io.Reader) error {
|
||||
log.Debugf("output ISO: %s %s", image, filename)
|
||||
log.Infof(" %s", filename)
|
||||
output, err := os.Create(filename)
|
||||
@ -296,10 +300,10 @@ func outputIso(image, filename string, filesystem []byte) error {
|
||||
return err
|
||||
}
|
||||
defer output.Close()
|
||||
return dockerRun(bytes.NewBuffer(filesystem), output, true, image)
|
||||
return dockerRun(filesystem, output, true, image)
|
||||
}
|
||||
|
||||
func outputRPi3(image, filename string, filesystem []byte) error {
|
||||
func outputRPi3(image, filename string, filesystem io.Reader) error {
|
||||
log.Debugf("output RPi3: %s %s", image, filename)
|
||||
log.Infof(" %s", filename)
|
||||
output, err := os.Create(filename)
|
||||
@ -307,7 +311,7 @@ func outputRPi3(image, filename string, filesystem []byte) error {
|
||||
return err
|
||||
}
|
||||
defer output.Close()
|
||||
return dockerRun(bytes.NewBuffer(filesystem), output, true, image)
|
||||
return dockerRun(filesystem, output, true, image)
|
||||
}
|
||||
|
||||
func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline string) error {
|
||||
|
Loading…
Reference in New Issue
Block a user