github/linuxkit
1
0
Fork 0
mirror of https://github.com/linuxkit/linuxkit.git synced 2025-07-19 17:26:28 +00:00
Code Issues Projects Releases Wiki Activity

Generate intermediate image into a temp file

Browse Source 
All of the `output*` functions took a `[]byte` and immediately wrapped it in a
`bytes.Buffer` to produce an `io.Reader`. Make them take an `io.Reader` instead
and satisfy this further up the call chain by directing `moby.Build` to output
to a temp file instead of another `bytes.Buffer`.

In my test case (building kube master image) this reduces Maximum RSS (as
measured by time(1)) from 6.7G to 2.8G and overall allocations from 9.7G to
5.3G. When building a tar (output to /dev/null) the Maximum RSS fell slightly
from 2.2G to 2.1G. Overall allocations remained stable at around 5.3G.

Signed-off-by: Ian Campbell <ijc@docker.com>
This commit is contained in:
Ian Campbell 2017-12-06 14:53:18 +00:00
parent 9558740c11
commit 9f44acf8e3
3 changed files with 54 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/moby/build.go
View File

@ -185,14 +185,18 @@ func build(args []string) {
m.Trust = moby.TrustConfig{}
}
var buf *bytes.Buffer
var tf *os.File
var w io.Writer
if outputFile != nil {
w = outputFile
} else {
buf = new(bytes.Buffer)
w = buf
if tf, err = ioutil.TempFile("", ""); err != nil {
log.Fatalf("Error creating tempfile: %v", err)
}
defer os.Remove(tf.Name())
w = tf
}
// this is a weird interface, but currently only streamable types can have additional files
// need to split up the base tarball outputs from the secondary stages
var tp string
@ -205,7 +209,11 @@ func build(args []string) {
}
if outputFile == nil {
image := buf.Bytes()
image := tf.Name()
if err := tf.Close(); err != nil {
log.Fatalf("Error closing tempfile: %v", err)
}
log.Infof("Create outputs:")
err = moby.Formats(filepath.Join(*buildDir, name), image, buildFormats, size)
if err != nil {

19
src/moby/linuxkit.go
View File

@ -1,7 +1,6 @@
package moby
import (
"bytes"
"crypto/sha256"
"fmt"
"io"
@ -58,9 +57,21 @@ func ensureLinuxkitImage(name string) error {
return err
}
// TODO pass through --pull to here
buf := new(bytes.Buffer)
Build(m, buf, false, "")
image := buf.Bytes()
tf, err := ioutil.TempFile("", "")
if err != nil {
return err
}
defer os.Remove(tf.Name())
Build(m, tf, false, "")
if err := tf.Close(); err != nil {
return err
}
image, err := os.Open(tf.Name())
if err != nil {
return err
}
defer image.Close()
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)

50
src/moby/output.go
View File

@ -4,6 +4,7 @@ import (
"archive/tar"
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"runtime"
@ -24,8 +25,8 @@ const (
rpi3 = "linuxkit/mkimage-rpi3:0735656fff247ca978135e3aeb62864adc612180"
)
var outFuns = map[string]func(string, []byte, int) error{
"kernel+initrd": func(base string, image []byte, size int) error {
var outFuns = map[string]func(string, io.Reader, int) error{
"kernel+initrd": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -36,7 +37,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"tar-kernel-initrd": func(base string, image []byte, size int) error {
"tar-kernel-initrd": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -46,21 +47,21 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"iso-bios": func(base string, image []byte, size int) error {
"iso-bios": func(base string, image io.Reader, size int) error {
err := outputIso(isoBios, base+".iso", image)
if err != nil {
return fmt.Errorf("Error writing iso-bios output: %v", err)
}
return nil
},
"iso-efi": func(base string, image []byte, size int) error {
"iso-efi": func(base string, image io.Reader, size int) error {
err := outputIso(isoEfi, base+"-efi.iso", image)
if err != nil {
return fmt.Errorf("Error writing iso-efi output: %v", err)
}
return nil
},
"raw-bios": func(base string, image []byte, size int) error {
"raw-bios": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -71,7 +72,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"raw-efi": func(base string, image []byte, size int) error {
"raw-efi": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -82,7 +83,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"aws": func(base string, image []byte, size int) error {
"aws": func(base string, image io.Reader, size int) error {
filename := base + ".raw"
log.Infof(" %s", filename)
kernel, initrd, cmdline, err := tarToInitrd(image)
@ -95,7 +96,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"gcp": func(base string, image []byte, size int) error {
"gcp": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -106,7 +107,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"qcow2-bios": func(base string, image []byte, size int) error {
"qcow2-bios": func(base string, image io.Reader, size int) error {
filename := base + ".qcow2"
log.Infof(" %s", filename)
kernel, initrd, cmdline, err := tarToInitrd(image)
@ -119,7 +120,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"vhd": func(base string, image []byte, size int) error {
"vhd": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -130,7 +131,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"dynamic-vhd": func(base string, image []byte, size int) error {
"dynamic-vhd": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -141,7 +142,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"vmdk": func(base string, image []byte, size int) error {
"vmdk": func(base string, image io.Reader, size int) error {
kernel, initrd, cmdline, err := tarToInitrd(image)
if err != nil {
return fmt.Errorf("Error converting to initrd: %v", err)
@ -152,7 +153,7 @@ var outFuns = map[string]func(string, []byte, int) error{
}
return nil
},
"rpi3": func(base string, image []byte, size int) error {
"rpi3": func(base string, image io.Reader, size int) error {
if runtime.GOARCH != "arm64" {
return fmt.Errorf("Raspberry Pi output currently only supported on arm64")
}
@ -197,7 +198,7 @@ func ValidateFormats(formats []string) error {
}
// Formats generates all the specified output formats
func Formats(base string, image []byte, formats []string, size int) error {
func Formats(base string, image string, formats []string, size int) error {
log.Debugf("format: %v %s", formats, base)
err := ValidateFormats(formats)
@ -205,20 +206,23 @@ func Formats(base string, image []byte, formats []string, size int) error {
return err
}
for _, o := range formats {
f := outFuns[o]
err := f(base, image, size)
ir, err := os.Open(image)
if err != nil {
return err
}
defer ir.Close()
f := outFuns[o]
if err := f(base, ir, size); err != nil {
return err
}
}
return nil
}
func tarToInitrd(image []byte) ([]byte, []byte, string, error) {
func tarToInitrd(r io.Reader) ([]byte, []byte, string, error) {
w := new(bytes.Buffer)
iw := initrd.NewWriter(w)
r := bytes.NewReader(image)
tr := tar.NewReader(r)
kernel, cmdline, err := initrd.CopySplitTar(iw, tr)
if err != nil {
@ -288,7 +292,7 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str
return dockerRun(buf, output, true, image, cmdline)
}
func outputIso(image, filename string, filesystem []byte) error {
func outputIso(image, filename string, filesystem io.Reader) error {
log.Debugf("output ISO: %s %s", image, filename)
log.Infof(" %s", filename)
output, err := os.Create(filename)
@ -296,10 +300,10 @@ func outputIso(image, filename string, filesystem []byte) error {
return err
}
defer output.Close()
return dockerRun(bytes.NewBuffer(filesystem), output, true, image)
return dockerRun(filesystem, output, true, image)
}
func outputRPi3(image, filename string, filesystem []byte) error {
func outputRPi3(image, filename string, filesystem io.Reader) error {
log.Debugf("output RPi3: %s %s", image, filename)
log.Infof(" %s", filename)
output, err := os.Create(filename)
@ -307,7 +311,7 @@ func outputRPi3(image, filename string, filesystem []byte) error {
return err
}
defer output.Close()
return dockerRun(bytes.NewBuffer(filesystem), output, true, image)
return dockerRun(filesystem, output, true, image)
}
func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline string) error {