From b086231008d9f2d2817380e0b00c1a8b9f4624e0 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Tue, 6 Jun 2017 14:34:19 +0100 Subject: [PATCH] Allow streaming output for tar This is a little ugly in terms of the validation now, but it is a move towards splitting "build" and "package". The "tar" output (and soon others) can output direct to a file or to stdout. Obviously you can only build a single output format like this. The LinuxKit output formats that build disk images cannot stream as they have to build whole images. These allow multiple outputs. In future we will probably change to ``` moby build | moby package ``` or similar, but that is a bit ugly, so currently have a compromise where there are essentially two output types. Signed-off-by: Justin Cormack --- cmd/moby/build.go | 119 ++++++++++++++++++++++++++++++++++--------- cmd/moby/linuxkit.go | 5 +- cmd/moby/output.go | 13 ----- 3 files changed, 98 insertions(+), 39 deletions(-) diff --git a/cmd/moby/build.go b/cmd/moby/build.go index b01f3c95a..8a1d5afb0 100644 --- a/cmd/moby/build.go +++ b/cmd/moby/build.go @@ -33,11 +33,18 @@ func (o *outputList) Set(value string) error { return nil } +var streamable = map[string]bool{ + "tar": true, +} + // Process the build arguments and execute build func build(args []string) { var buildOut outputList outputTypes := []string{} + for k := range streamable { + outputTypes = append(outputTypes, k) + } for k := range outFuns { outputTypes = append(outputTypes, k) } @@ -51,6 +58,7 @@ func build(args []string) { } buildName := buildCmd.String("name", "", "Name to use for output files") buildDir := buildCmd.String("dir", "", "Directory for output files, default current directory") + buildOutputFile := buildCmd.String("o", "", "File to use for a single output, or '-' for stdout") buildSize := buildCmd.String("size", "1024M", "Size for output image, if supported and fixed size") buildPull := buildCmd.Bool("pull", false, "Always pull images") buildDisableTrust := buildCmd.Bool("disable-content-trust", false, "Skip image trust verification specified in trust section of config (default false)") @@ -68,17 +76,79 @@ func build(args []string) { os.Exit(1) } + name := *buildName + if name == "" { + conf := remArgs[len(remArgs)-1] + if conf == "-" { + name = defaultNameForStdin + } else { + name = strings.TrimSuffix(filepath.Base(conf), filepath.Ext(conf)) + } + } + + // There are two types of output, they will probably be split into "build" and "package" later + // the basic outputs are tarballs, while the packaged ones are the LinuxKit out formats that + // cannot be streamed but we do allow multiple ones to be built. + if len(buildOut) == 0 { - buildOut = outputList{"kernel+initrd"} + if *buildOutputFile == "" { + buildOut = outputList{"kernel+initrd"} + } else { + buildOut = outputList{"tar"} + } } log.Debugf("Outputs selected: %s", buildOut.String()) - err := validateOutputs(buildOut) - if err != nil { - log.Errorf("Error parsing outputs: %v", err) - buildCmd.Usage() - os.Exit(1) + if len(buildOut) > 1 { + for _, o := range buildOut { + if streamable[o] { + log.Fatalf("Output type %s must be the only output specified", o) + } + } + } + + if len(buildOut) == 1 && streamable[buildOut[0]] { + if *buildOutputFile == "" { + *buildOutputFile = filepath.Join(*buildDir, name+"."+buildOut[0]) + // stop the erros in the validation below + *buildName = "" + *buildDir = "" + } + + } else { + err := validateOutputs(buildOut) + if err != nil { + log.Errorf("Error parsing outputs: %v", err) + buildCmd.Usage() + os.Exit(1) + } + } + + var outputFile *os.File + if *buildOutputFile != "" { + if len(buildOut) > 1 { + log.Fatal("The -output option can only be specified when generating a single output format") + } + if *buildName != "" { + log.Fatal("The -output option cannot be specified with -name") + } + if *buildDir != "" { + log.Fatal("The -output option cannot be specified with -dir") + } + if !streamable[buildOut[0]] { + log.Fatalf("The -output option cannot be specified for build type %s as it cannot be streamed", buildOut[0]) + } + if *buildOutputFile == "-" { + outputFile = os.Stdout + } else { + var err error + outputFile, err = os.Create(*buildOutputFile) + if err != nil { + log.Fatalf("Cannot open output file: %v", err) + } + defer outputFile.Close() + } } size, err := getDiskSizeMB(*buildSize) @@ -86,8 +156,6 @@ func build(args []string) { log.Fatalf("Unable to parse disk size: %v", err) } - name := *buildName - var moby Moby for _, arg := range remArgs { var config []byte @@ -97,21 +165,12 @@ func build(args []string) { if err != nil { log.Fatalf("Cannot read stdin: %v", err) } - if name == "" { - name = defaultNameForStdin - } } else { - if !(filepath.Ext(conf) == ".yml" || filepath.Ext(conf) == ".yaml") { - conf = conf + ".yml" - } var err error config, err = ioutil.ReadFile(conf) if err != nil { log.Fatalf("Cannot open config file: %v", err) } - if name == "" { - name = strings.TrimSuffix(filepath.Base(conf), filepath.Ext(conf)) - } } m, err := NewConfig(config) @@ -126,12 +185,23 @@ func build(args []string) { moby.Trust = TrustConfig{} } - image := buildInternal(moby, *buildPull) + var buf *bytes.Buffer + var w io.Writer + if outputFile != nil { + w = outputFile + } else { + buf = new(bytes.Buffer) + w = buf + } + buildInternal(moby, w, *buildPull) - log.Infof("Create outputs:") - err = outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit) - if err != nil { - log.Fatalf("Error writing outputs: %v", err) + if outputFile == nil { + image := buf.Bytes() + log.Infof("Create outputs:") + err = outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit) + if err != nil { + log.Fatalf("Error writing outputs: %v", err) + } } } @@ -222,8 +292,7 @@ func enforceContentTrust(fullImageName string, config *TrustConfig) bool { // Perform the actual build process // TODO return error not panic -func buildInternal(m Moby, pull bool) []byte { - w := new(bytes.Buffer) +func buildInternal(m Moby, w io.Writer, pull bool) { iw := tar.NewWriter(w) if m.Kernel.Image != "" { @@ -312,7 +381,7 @@ func buildInternal(m Moby, pull bool) []byte { log.Fatalf("initrd close error: %v", err) } - return w.Bytes() + return } func untarKernel(buf *bytes.Buffer, kernelName, kernelAltName, ktarName string, cmdline string) (*bytes.Buffer, *bytes.Buffer, error) { diff --git a/cmd/moby/linuxkit.go b/cmd/moby/linuxkit.go index 5d14e5855..e4d2a9f1b 100644 --- a/cmd/moby/linuxkit.go +++ b/cmd/moby/linuxkit.go @@ -1,6 +1,7 @@ package main import ( + "bytes" "crypto/sha256" "fmt" "io" @@ -58,7 +59,9 @@ func ensureLinuxkitImage(name string) error { return err } // TODO pass through --pull to here - image := buildInternal(m, false) + buf := new(bytes.Buffer) + buildInternal(m, buf, false) + image := buf.Bytes() kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { return fmt.Errorf("Error converting to initrd: %v", err) diff --git a/cmd/moby/output.go b/cmd/moby/output.go index 6d7843e50..9a0c7358c 100644 --- a/cmd/moby/output.go +++ b/cmd/moby/output.go @@ -23,13 +23,6 @@ const ( ) var outFuns = map[string]func(string, []byte, int, bool) error{ - "tar": func(base string, image []byte, size int, hyperkit bool) error { - err := outputTar(base, image) - if err != nil { - return fmt.Errorf("Error writing tar output: %v", err) - } - return nil - }, "kernel+initrd": func(base string, image []byte, size int, hyperkit bool) error { kernel, initrd, cmdline, err := tarToInitrd(image) if err != nil { @@ -336,9 +329,3 @@ func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline strin } return nil } - -func outputTar(base string, initrd []byte) error { - log.Debugf("output tar: %s", base) - log.Infof(" %s", base+".tar") - return ioutil.WriteFile(base+".tar", initrd, os.FileMode(0644)) -}