Split out into a small stub command line and a library

- this is pretty much the smallest change to split this out and it
  exposes a few things that can be improved later
- no change to logging yet

Signed-off-by: Justin Cormack <justin.cormack@docker.com>

Makefile

@@ -18,7 +18,7 @@ lint:
 	# govet
 	@test -z "$$(go tool vet -printf=false . 2>&1 | grep -v vendor/ | tee /dev/stderr)"
 	# go test
-	@go test github.com/moby/tool/cmd/moby
+	@go test github.com/moby/tool/src/moby
 
 test: moby
 	./moby build -output tar test/test.yml

cmd/moby/build.go

@@ -1,23 +1,20 @@
 package main
 
 import (
-	"archive/tar"
 	"bytes"
-	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"path"
 	"path/filepath"
 	"runtime"
-	"sort"
 	"strconv"
 	"strings"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/moby/tool/src/moby"
 )
 
 const defaultNameForStdin = "moby"
@@ -36,52 +33,32 @@ func (o *outputList) Set(value string) error {
 	return nil
 }
 
-var streamable = map[string]bool{
-	"docker": true,
-	"tar":    true,
-}
-
-type addFun func(*tar.Writer) error
-
-const dockerfile = `
-FROM scratch
-
-COPY . ./
-RUN rm -f Dockerfile
-
-ENTRYPOINT ["/sbin/tini", "--", "/bin/rc.init"]
-`
-
-var additions = map[string]addFun{
-	"docker": func(tw *tar.Writer) error {
-		log.Infof("  Adding Dockerfile")
-		hdr := &tar.Header{
-			Name: "Dockerfile",
-			Mode: 0644,
-			Size: int64(len(dockerfile)),
+// Parse a string which is either a number in MB, or a number with
+// either M (for Megabytes) or G (for GigaBytes) as a suffix and
+// returns the number in MB. Return 0 if string is empty.
+func getDiskSizeMB(s string) (int, error) {
+	if s == "" {
+		return 0, nil
+	}
+	sz := len(s)
+	if strings.HasSuffix(s, "G") {
+		i, err := strconv.Atoi(s[:sz-1])
+		if err != nil {
+			return 0, err
 		}
-		if err := tw.WriteHeader(hdr); err != nil {
-			return err
-		}
-		if _, err := tw.Write([]byte(dockerfile)); err != nil {
-			return err
-		}
-		return nil
-	},
+		return i * 1024, nil
+	}
+	if strings.HasSuffix(s, "M") {
+		s = s[:sz-1]
+	}
+	return strconv.Atoi(s)
 }
 
 // Process the build arguments and execute build
 func build(args []string) {
 	var buildOut outputList
 
-	outputTypes := []string{}
-	for k := range streamable {
-		outputTypes = append(outputTypes, k)
-	}
-	for k := range outFuns {
-		outputTypes = append(outputTypes, k)
-	}
-	sort.Strings(outputTypes)
+	outputTypes := moby.OutputTypes()
 
 	buildCmd := flag.NewFlagSet("build", flag.ExitOnError)
 	buildCmd.Usage = func() {
@@ -135,13 +112,13 @@ func build(args []string) {
 
 	if len(buildOut) > 1 {
 		for _, o := range buildOut {
-			if streamable[o] {
+			if moby.Streamable(o) {
 				log.Fatalf("Output type %s must be the only output specified", o)
 			}
 		}
 	}
 
-	if len(buildOut) == 1 && streamable[buildOut[0]] {
+	if len(buildOut) == 1 && moby.Streamable(buildOut[0]) {
 		if *buildOutputFile == "" {
 			*buildOutputFile = filepath.Join(*buildDir, name+"."+buildOut[0])
 			// stop the errors in the validation below
@@ -150,7 +127,7 @@ func build(args []string) {
 		}
 
 	} else {
-		err := validateOutputs(buildOut)
+		err := moby.ValidateOutputs(buildOut)
 		if err != nil {
 			log.Errorf("Error parsing outputs: %v", err)
 			buildCmd.Usage()
@@ -159,7 +136,6 @@ func build(args []string) {
 	}
 
 	var outputFile *os.File
-	var addition addFun
 	if *buildOutputFile != "" {
 		if len(buildOut) > 1 {
 			log.Fatal("The -output option can only be specified when generating a single output format")
@@ -170,7 +146,7 @@ func build(args []string) {
 		if *buildDir != "" {
 			log.Fatal("The -output option cannot be specified with -dir")
 		}
-		if !streamable[buildOut[0]] {
+		if !moby.Streamable(buildOut[0]) {
 			log.Fatalf("The -output option cannot be specified for build type %s as it cannot be streamed", buildOut[0])
 		}
 		if *buildOutputFile == "-" {
@@ -183,7 +159,6 @@ func build(args []string) {
 			}
 			defer outputFile.Close()
 		}
-		addition = additions[buildOut[0]]
 	}
 
 	size, err := getDiskSizeMB(*buildSize)
@@ -191,7 +166,7 @@ func build(args []string) {
 		log.Fatalf("Unable to parse disk size: %v", err)
 	}
 
-	var moby Moby
+	var m moby.Moby
 	for _, arg := range remArgs {
 		var config []byte
 		if conf := arg; conf == "-" {
@@ -220,16 +195,16 @@ func build(args []string) {
 			}
 		}
 
-		m, err := NewConfig(config)
+		c, err := moby.NewConfig(config)
 		if err != nil {
 			log.Fatalf("Invalid config: %v", err)
 		}
-		moby = AppendConfig(moby, m)
+		m = moby.AppendConfig(m, c)
 	}
 
 	if *buildDisableTrust {
 		log.Debugf("Disabling content trust checks for this build")
-		moby.Trust = TrustConfig{}
+		m.Trust = moby.TrustConfig{}
 	}
 
 	var buf *bytes.Buffer
@@ -240,7 +215,13 @@ func build(args []string) {
 		buf = new(bytes.Buffer)
 		w = buf
 	}
-	err = buildInternal(moby, w, *buildPull, addition)
+	// this is a weird interface, but currently only streamable types can have additional files
+	// need to split up the base tarball outputs from the secondary stages
+	var tp string
+	if moby.Streamable(buildOut[0]) {
+		tp = buildOut[0]
+	}
+	err = moby.Build(m, w, *buildPull, tp)
 	if err != nil {
 		log.Fatalf("%v", err)
 	}
@@ -248,418 +229,9 @@ func build(args []string) {
 	if outputFile == nil {
 		image := buf.Bytes()
 		log.Infof("Create outputs:")
-		err = outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit)
+		err = moby.Outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit)
 		if err != nil {
 			log.Fatalf("Error writing outputs: %v", err)
 		}
 	}
 }
-
-// Parse a string which is either a number in MB, or a number with
-// either M (for Megabytes) or G (for GigaBytes) as a suffix and
-// returns the number in MB. Return 0 if string is empty.
-func getDiskSizeMB(s string) (int, error) {
-	if s == "" {
-		return 0, nil
-	}
-	sz := len(s)
-	if strings.HasSuffix(s, "G") {
-		i, err := strconv.Atoi(s[:sz-1])
-		if err != nil {
-			return 0, err
-		}
-		return i * 1024, nil
-	}
-	if strings.HasSuffix(s, "M") {
-		s = s[:sz-1]
-	}
-	return strconv.Atoi(s)
-}
-
-func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
-	for _, img := range config.Image {
-		// First check for an exact name match
-		if img == fullImageName {
-			return true
-		}
-		// Also check for an image name only match
-		// by removing a possible tag (with possibly added digest):
-		imgAndTag := strings.Split(fullImageName, ":")
-		if len(imgAndTag) >= 2 && img == imgAndTag[0] {
-			return true
-		}
-		// and by removing a possible digest:
-		imgAndDigest := strings.Split(fullImageName, "@sha256:")
-		if len(imgAndDigest) >= 2 && img == imgAndDigest[0] {
-			return true
-		}
-	}
-
-	for _, org := range config.Org {
-		var imgOrg string
-		splitName := strings.Split(fullImageName, "/")
-		switch len(splitName) {
-		case 0:
-			// if the image is empty, return false
-			return false
-		case 1:
-			// for single names like nginx, use library
-			imgOrg = "library"
-		case 2:
-			// for names that assume docker hub, like linxukit/alpine, take the first split
-			imgOrg = splitName[0]
-		default:
-			// for names that include the registry, the second piece is the org, ex: docker.io/library/alpine
-			imgOrg = splitName[1]
-		}
-		if imgOrg == org {
-			return true
-		}
-	}
-	return false
-}
-
-// Perform the actual build process
-func buildInternal(m Moby, w io.Writer, pull bool, addition addFun) error {
-	iw := tar.NewWriter(w)
-
-	if m.Kernel.Image != "" {
-		// get kernel and initrd tarball from container
-		log.Infof("Extract kernel image: %s", m.Kernel.Image)
-		kf := newKernelFilter(iw, m.Kernel.Cmdline)
-		err := ImageTar(m.Kernel.Image, "", kf, enforceContentTrust(m.Kernel.Image, &m.Trust), pull)
-		if err != nil {
-			return fmt.Errorf("Failed to extract kernel image and tarball: %v", err)
-		}
-		err = kf.Close()
-		if err != nil {
-			return fmt.Errorf("Close error: %v", err)
-		}
-	}
-
-	// convert init images to tarballs
-	if len(m.Init) != 0 {
-		log.Infof("Add init containers:")
-	}
-	for _, ii := range m.Init {
-		log.Infof("Process init image: %s", ii)
-		err := ImageTar(ii, "", iw, enforceContentTrust(ii, &m.Trust), pull)
-		if err != nil {
-			return fmt.Errorf("Failed to build init tarball from %s: %v", ii, err)
-		}
-	}
-
-	if len(m.Onboot) != 0 {
-		log.Infof("Add onboot containers:")
-	}
-	for i, image := range m.Onboot {
-		log.Infof("  Create OCI config for %s", image.Image)
-		useTrust := enforceContentTrust(image.Image, &m.Trust)
-		config, err := ConfigToOCI(image, useTrust)
-		if err != nil {
-			return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err)
-		}
-		so := fmt.Sprintf("%03d", i)
-		path := "containers/onboot/" + so + "-" + image.Name
-		err = ImageBundle(path, image.Image, config, iw, useTrust, pull)
-		if err != nil {
-			return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err)
-		}
-	}
-
-	if len(m.Services) != 0 {
-		log.Infof("Add service containers:")
-	}
-	for _, image := range m.Services {
-		log.Infof("  Create OCI config for %s", image.Image)
-		useTrust := enforceContentTrust(image.Image, &m.Trust)
-		config, err := ConfigToOCI(image, useTrust)
-		if err != nil {
-			return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err)
-		}
-		path := "containers/services/" + image.Name
-		err = ImageBundle(path, image.Image, config, iw, useTrust, pull)
-		if err != nil {
-			return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err)
-		}
-	}
-
-	// add files
-	err := filesystem(m, iw)
-	if err != nil {
-		return fmt.Errorf("failed to add filesystem parts: %v", err)
-	}
-
-	// add anything additional for this output type
-	if addition != nil {
-		err = addition(iw)
-		if err != nil {
-			return fmt.Errorf("Failed to add additional files: %v", err)
-		}
-	}
-
-	err = iw.Close()
-	if err != nil {
-		return fmt.Errorf("initrd close error: %v", err)
-	}
-
-	return nil
-}
-
-// kernelFilter is a tar.Writer that transforms a kernel image into the output we want on underlying tar writer
-type kernelFilter struct {
-	tw          *tar.Writer
-	buffer      *bytes.Buffer
-	cmdline     string
-	discard     bool
-	foundKernel bool
-	foundKTar   bool
-}
-
-func newKernelFilter(tw *tar.Writer, cmdline string) *kernelFilter {
-	return &kernelFilter{tw: tw, cmdline: cmdline}
-}
-
-func (k *kernelFilter) finishTar() error {
-	if k.buffer == nil {
-		return nil
-	}
-	tr := tar.NewReader(k.buffer)
-	err := tarAppend(k.tw, tr)
-	k.buffer = nil
-	return err
-}
-
-func (k *kernelFilter) Close() error {
-	if !k.foundKernel {
-		return errors.New("did not find kernel in kernel image")
-	}
-	if !k.foundKTar {
-		return errors.New("did not find kernel.tar in kernel image")
-	}
-	return k.finishTar()
-}
-
-func (k *kernelFilter) Flush() error {
-	err := k.finishTar()
-	if err != nil {
-		return err
-	}
-	return k.tw.Flush()
-}
-
-func (k *kernelFilter) Write(b []byte) (n int, err error) {
-	if k.discard {
-		return len(b), nil
-	}
-	if k.buffer != nil {
-		return k.buffer.Write(b)
-	}
-	return k.tw.Write(b)
-}
-
-func (k *kernelFilter) WriteHeader(hdr *tar.Header) error {
-	err := k.finishTar()
-	if err != nil {
-		return err
-	}
-	tw := k.tw
-	switch hdr.Name {
-	case "kernel":
-		if k.foundKernel {
-			return errors.New("found more than one possible kernel image")
-		}
-		k.foundKernel = true
-		k.discard = false
-		whdr := &tar.Header{
-			Name:     "boot",
-			Mode:     0755,
-			Typeflag: tar.TypeDir,
-		}
-		if err := tw.WriteHeader(whdr); err != nil {
-			return err
-		}
-		// add the cmdline in /boot/cmdline
-		whdr = &tar.Header{
-			Name: "boot/cmdline",
-			Mode: 0644,
-			Size: int64(len(k.cmdline)),
-		}
-		if err := tw.WriteHeader(whdr); err != nil {
-			return err
-		}
-		buf := bytes.NewBufferString(k.cmdline)
-		_, err = io.Copy(tw, buf)
-		if err != nil {
-			return err
-		}
-		whdr = &tar.Header{
-			Name: "boot/kernel",
-			Mode: hdr.Mode,
-			Size: hdr.Size,
-		}
-		if err := tw.WriteHeader(whdr); err != nil {
-			return err
-		}
-	case "kernel.tar":
-		k.foundKTar = true
-		k.discard = false
-		k.buffer = new(bytes.Buffer)
-	default:
-		k.discard = true
-	}
-
-	return nil
-}
-
-func tarAppend(iw *tar.Writer, tr *tar.Reader) error {
-	for {
-		hdr, err := tr.Next()
-		if err == io.EOF {
-			break
-		}
-		if err != nil {
-			return err
-		}
-		err = iw.WriteHeader(hdr)
-		if err != nil {
-			return err
-		}
-		_, err = io.Copy(iw, tr)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func filesystem(m Moby, tw *tar.Writer) error {
-	// TODO also include the files added in other parts of the build
-	var addedFiles = map[string]bool{}
-
-	if len(m.Files) != 0 {
-		log.Infof("Add files:")
-	}
-	for _, f := range m.Files {
-		log.Infof("  %s", f.Path)
-		if f.Path == "" {
-			return errors.New("Did not specify path for file")
-		}
-		// tar archives should not have absolute paths
-		if f.Path[0] == os.PathSeparator {
-			f.Path = f.Path[1:]
-		}
-		mode := int64(0600)
-		if f.Directory {
-			mode = 0700
-		}
-		if f.Mode != "" {
-			var err error
-			mode, err = strconv.ParseInt(f.Mode, 8, 32)
-			if err != nil {
-				return fmt.Errorf("Cannot parse file mode as octal value: %v", err)
-			}
-		}
-		dirMode := mode
-		if dirMode&0700 != 0 {
-			dirMode |= 0100
-		}
-		if dirMode&0070 != 0 {
-			dirMode |= 0010
-		}
-		if dirMode&0007 != 0 {
-			dirMode |= 0001
-		}
-		var contents []byte
-		if f.Contents != nil {
-			contents = []byte(*f.Contents)
-		}
-		if !f.Directory && f.Contents == nil && f.Symlink == "" {
-			if f.Source == "" {
-				return errors.New("Contents of file not specified")
-			}
-			if len(f.Source) > 2 && f.Source[:2] == "~/" {
-				f.Source = homeDir() + f.Source[1:]
-			}
-			if f.Optional {
-				_, err := os.Stat(f.Source)
-				if err != nil {
-					// skip if not found or readable
-					log.Debugf("Skipping file [%s] as not readable and marked optional", f.Source)
-					continue
-				}
-			}
-			var err error
-			contents, err = ioutil.ReadFile(f.Source)
-			if err != nil {
-				return err
-			}
-		}
-		// we need all the leading directories
-		parts := strings.Split(path.Dir(f.Path), "/")
-		root := ""
-		for _, p := range parts {
-			if p == "." || p == "/" {
-				continue
-			}
-			if root == "" {
-				root = p
-			} else {
-				root = root + "/" + p
-			}
-			if !addedFiles[root] {
-				hdr := &tar.Header{
-					Name:     root,
-					Typeflag: tar.TypeDir,
-					Mode:     dirMode,
-				}
-				err := tw.WriteHeader(hdr)
-				if err != nil {
-					return err
-				}
-				addedFiles[root] = true
-			}
-		}
-		addedFiles[f.Path] = true
-		if f.Directory {
-			if f.Contents != nil {
-				return errors.New("Directory with contents not allowed")
-			}
-			hdr := &tar.Header{
-				Name:     f.Path,
-				Typeflag: tar.TypeDir,
-				Mode:     mode,
-			}
-			err := tw.WriteHeader(hdr)
-			if err != nil {
-				return err
-			}
-		} else if f.Symlink != "" {
-			hdr := &tar.Header{
-				Name:     f.Path,
-				Typeflag: tar.TypeSymlink,
-				Mode:     mode,
-				Linkname: f.Symlink,
-			}
-			err := tw.WriteHeader(hdr)
-			if err != nil {
-				return err
-			}
-		} else {
-			hdr := &tar.Header{
-				Name: f.Path,
-				Mode: mode,
-				Size: int64(len(contents)),
-			}
-			err := tw.WriteHeader(hdr)
-			if err != nil {
-				return err
-			}
-			_, err = tw.Write(contents)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}

cmd/moby/main.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	log "github.com/Sirupsen/logrus"
+	"github.com/moby/tool/src/moby"
 )
 
 var (
@@ -17,9 +18,6 @@ var (
 
 	// GitCommit hash, set at compile time
 	GitCommit = "unknown"
-
-	// MobyDir is the location of the cache directory ~/.moby by default
-	MobyDir string
 )
 
 // infoFormatter overrides the default format for Info() log events to
@@ -91,16 +89,17 @@ func main() {
 		os.Exit(1)
 	}
 
-	MobyDir = *flagConfigDir
-	err := os.MkdirAll(MobyDir, 0755)
+	mobyDir := *flagConfigDir
+	err := os.MkdirAll(mobyDir, 0755)
 	if err != nil {
-		log.Fatalf("Could not create config directory [%s]: %v", MobyDir, err)
+		log.Fatalf("Could not create config directory [%s]: %v", mobyDir, err)
 	}
 
-	err = os.MkdirAll(filepath.Join(MobyDir, "tmp"), 0755)
+	err = os.MkdirAll(filepath.Join(mobyDir, "tmp"), 0755)
 	if err != nil {
-		log.Fatalf("Could not create config tmp directory [%s]: %v", filepath.Join(MobyDir, "tmp"), err)
+		log.Fatalf("Could not create config tmp directory [%s]: %v", filepath.Join(mobyDir, "tmp"), err)
 	}
+	moby.MobyDir = mobyDir
 
 	switch args[0] {
 	case "build":

src/moby/build.go

@@ -0,0 +1,467 @@
+package moby
+
+import (
+	"archive/tar"
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strconv"
+	"strings"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+const defaultNameForStdin = "moby"
+
+var streamable = map[string]bool{
+	"docker": true,
+	"tar":    true,
+}
+
+// Streamable returns true if an output can be streamed
+func Streamable(t string) bool {
+	return streamable[t]
+}
+
+type addFun func(*tar.Writer) error
+
+const dockerfile = `
+FROM scratch
+
+COPY . ./
+RUN rm -f Dockerfile
+
+ENTRYPOINT ["/sbin/tini", "--", "/bin/rc.init"]
+`
+
+var additions = map[string]addFun{
+	"docker": func(tw *tar.Writer) error {
+		log.Infof("  Adding Dockerfile")
+		hdr := &tar.Header{
+			Name: "Dockerfile",
+			Mode: 0644,
+			Size: int64(len(dockerfile)),
+		}
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if _, err := tw.Write([]byte(dockerfile)); err != nil {
+			return err
+		}
+		return nil
+	},
+}
+
+// OutputTypes returns a list of the valid output types
+func OutputTypes() []string {
+	ts := []string{}
+	for k := range streamable {
+		ts = append(ts, k)
+	}
+	for k := range outFuns {
+		ts = append(ts, k)
+	}
+	sort.Strings(ts)
+
+	return ts
+}
+
+func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
+	for _, img := range config.Image {
+		// First check for an exact name match
+		if img == fullImageName {
+			return true
+		}
+		// Also check for an image name only match
+		// by removing a possible tag (with possibly added digest):
+		imgAndTag := strings.Split(fullImageName, ":")
+		if len(imgAndTag) >= 2 && img == imgAndTag[0] {
+			return true
+		}
+		// and by removing a possible digest:
+		imgAndDigest := strings.Split(fullImageName, "@sha256:")
+		if len(imgAndDigest) >= 2 && img == imgAndDigest[0] {
+			return true
+		}
+	}
+
+	for _, org := range config.Org {
+		var imgOrg string
+		splitName := strings.Split(fullImageName, "/")
+		switch len(splitName) {
+		case 0:
+			// if the image is empty, return false
+			return false
+		case 1:
+			// for single names like nginx, use library
+			imgOrg = "library"
+		case 2:
+			// for names that assume docker hub, like linxukit/alpine, take the first split
+			imgOrg = splitName[0]
+		default:
+			// for names that include the registry, the second piece is the org, ex: docker.io/library/alpine
+			imgOrg = splitName[1]
+		}
+		if imgOrg == org {
+			return true
+		}
+	}
+	return false
+}
+
+// Build performs the actual build process
+func Build(m Moby, w io.Writer, pull bool, tp string) error {
+	if MobyDir == "" {
+		return fmt.Errorf("MobyDir for temporary storage not set")
+	}
+
+	iw := tar.NewWriter(w)
+
+	// add additions
+	addition := additions[tp]
+
+	if m.Kernel.Image != "" {
+		// get kernel and initrd tarball from container
+		log.Infof("Extract kernel image: %s", m.Kernel.Image)
+		kf := newKernelFilter(iw, m.Kernel.Cmdline)
+		err := ImageTar(m.Kernel.Image, "", kf, enforceContentTrust(m.Kernel.Image, &m.Trust), pull)
+		if err != nil {
+			return fmt.Errorf("Failed to extract kernel image and tarball: %v", err)
+		}
+		err = kf.Close()
+		if err != nil {
+			return fmt.Errorf("Close error: %v", err)
+		}
+	}
+
+	// convert init images to tarballs
+	if len(m.Init) != 0 {
+		log.Infof("Add init containers:")
+	}
+	for _, ii := range m.Init {
+		log.Infof("Process init image: %s", ii)
+		err := ImageTar(ii, "", iw, enforceContentTrust(ii, &m.Trust), pull)
+		if err != nil {
+			return fmt.Errorf("Failed to build init tarball from %s: %v", ii, err)
+		}
+	}
+
+	if len(m.Onboot) != 0 {
+		log.Infof("Add onboot containers:")
+	}
+	for i, image := range m.Onboot {
+		log.Infof("  Create OCI config for %s", image.Image)
+		useTrust := enforceContentTrust(image.Image, &m.Trust)
+		config, err := ConfigToOCI(image, useTrust)
+		if err != nil {
+			return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err)
+		}
+		so := fmt.Sprintf("%03d", i)
+		path := "containers/onboot/" + so + "-" + image.Name
+		err = ImageBundle(path, image.Image, config, iw, useTrust, pull)
+		if err != nil {
+			return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err)
+		}
+	}
+
+	if len(m.Services) != 0 {
+		log.Infof("Add service containers:")
+	}
+	for _, image := range m.Services {
+		log.Infof("  Create OCI config for %s", image.Image)
+		useTrust := enforceContentTrust(image.Image, &m.Trust)
+		config, err := ConfigToOCI(image, useTrust)
+		if err != nil {
+			return fmt.Errorf("Failed to create config.json for %s: %v", image.Image, err)
+		}
+		path := "containers/services/" + image.Name
+		err = ImageBundle(path, image.Image, config, iw, useTrust, pull)
+		if err != nil {
+			return fmt.Errorf("Failed to extract root filesystem for %s: %v", image.Image, err)
+		}
+	}
+
+	// add files
+	err := filesystem(m, iw)
+	if err != nil {
+		return fmt.Errorf("failed to add filesystem parts: %v", err)
+	}
+
+	// add anything additional for this output type
+	if addition != nil {
+		err = addition(iw)
+		if err != nil {
+			return fmt.Errorf("Failed to add additional files: %v", err)
+		}
+	}
+
+	err = iw.Close()
+	if err != nil {
+		return fmt.Errorf("initrd close error: %v", err)
+	}
+
+	return nil
+}
+
+// kernelFilter is a tar.Writer that transforms a kernel image into the output we want on underlying tar writer
+type kernelFilter struct {
+	tw          *tar.Writer
+	buffer      *bytes.Buffer
+	cmdline     string
+	discard     bool
+	foundKernel bool
+	foundKTar   bool
+}
+
+func newKernelFilter(tw *tar.Writer, cmdline string) *kernelFilter {
+	return &kernelFilter{tw: tw, cmdline: cmdline}
+}
+
+func (k *kernelFilter) finishTar() error {
+	if k.buffer == nil {
+		return nil
+	}
+	tr := tar.NewReader(k.buffer)
+	err := tarAppend(k.tw, tr)
+	k.buffer = nil
+	return err
+}
+
+func (k *kernelFilter) Close() error {
+	if !k.foundKernel {
+		return errors.New("did not find kernel in kernel image")
+	}
+	if !k.foundKTar {
+		return errors.New("did not find kernel.tar in kernel image")
+	}
+	return k.finishTar()
+}
+
+func (k *kernelFilter) Flush() error {
+	err := k.finishTar()
+	if err != nil {
+		return err
+	}
+	return k.tw.Flush()
+}
+
+func (k *kernelFilter) Write(b []byte) (n int, err error) {
+	if k.discard {
+		return len(b), nil
+	}
+	if k.buffer != nil {
+		return k.buffer.Write(b)
+	}
+	return k.tw.Write(b)
+}
+
+func (k *kernelFilter) WriteHeader(hdr *tar.Header) error {
+	err := k.finishTar()
+	if err != nil {
+		return err
+	}
+	tw := k.tw
+	switch hdr.Name {
+	case "kernel":
+		if k.foundKernel {
+			return errors.New("found more than one possible kernel image")
+		}
+		k.foundKernel = true
+		k.discard = false
+		whdr := &tar.Header{
+			Name:     "boot",
+			Mode:     0755,
+			Typeflag: tar.TypeDir,
+		}
+		if err := tw.WriteHeader(whdr); err != nil {
+			return err
+		}
+		// add the cmdline in /boot/cmdline
+		whdr = &tar.Header{
+			Name: "boot/cmdline",
+			Mode: 0644,
+			Size: int64(len(k.cmdline)),
+		}
+		if err := tw.WriteHeader(whdr); err != nil {
+			return err
+		}
+		buf := bytes.NewBufferString(k.cmdline)
+		_, err = io.Copy(tw, buf)
+		if err != nil {
+			return err
+		}
+		whdr = &tar.Header{
+			Name: "boot/kernel",
+			Mode: hdr.Mode,
+			Size: hdr.Size,
+		}
+		if err := tw.WriteHeader(whdr); err != nil {
+			return err
+		}
+	case "kernel.tar":
+		k.foundKTar = true
+		k.discard = false
+		k.buffer = new(bytes.Buffer)
+	default:
+		k.discard = true
+	}
+
+	return nil
+}
+
+func tarAppend(iw *tar.Writer, tr *tar.Reader) error {
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		err = iw.WriteHeader(hdr)
+		if err != nil {
+			return err
+		}
+		_, err = io.Copy(iw, tr)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func filesystem(m Moby, tw *tar.Writer) error {
+	// TODO also include the files added in other parts of the build
+	var addedFiles = map[string]bool{}
+
+	if len(m.Files) != 0 {
+		log.Infof("Add files:")
+	}
+	for _, f := range m.Files {
+		log.Infof("  %s", f.Path)
+		if f.Path == "" {
+			return errors.New("Did not specify path for file")
+		}
+		// tar archives should not have absolute paths
+		if f.Path[0] == os.PathSeparator {
+			f.Path = f.Path[1:]
+		}
+		mode := int64(0600)
+		if f.Directory {
+			mode = 0700
+		}
+		if f.Mode != "" {
+			var err error
+			mode, err = strconv.ParseInt(f.Mode, 8, 32)
+			if err != nil {
+				return fmt.Errorf("Cannot parse file mode as octal value: %v", err)
+			}
+		}
+		dirMode := mode
+		if dirMode&0700 != 0 {
+			dirMode |= 0100
+		}
+		if dirMode&0070 != 0 {
+			dirMode |= 0010
+		}
+		if dirMode&0007 != 0 {
+			dirMode |= 0001
+		}
+		var contents []byte
+		if f.Contents != nil {
+			contents = []byte(*f.Contents)
+		}
+		if !f.Directory && f.Contents == nil && f.Symlink == "" {
+			if f.Source == "" {
+				return errors.New("Contents of file not specified")
+			}
+			if len(f.Source) > 2 && f.Source[:2] == "~/" {
+				f.Source = homeDir() + f.Source[1:]
+			}
+			if f.Optional {
+				_, err := os.Stat(f.Source)
+				if err != nil {
+					// skip if not found or readable
+					log.Debugf("Skipping file [%s] as not readable and marked optional", f.Source)
+					continue
+				}
+			}
+			var err error
+			contents, err = ioutil.ReadFile(f.Source)
+			if err != nil {
+				return err
+			}
+		}
+		// we need all the leading directories
+		parts := strings.Split(path.Dir(f.Path), "/")
+		root := ""
+		for _, p := range parts {
+			if p == "." || p == "/" {
+				continue
+			}
+			if root == "" {
+				root = p
+			} else {
+				root = root + "/" + p
+			}
+			if !addedFiles[root] {
+				hdr := &tar.Header{
+					Name:     root,
+					Typeflag: tar.TypeDir,
+					Mode:     dirMode,
+				}
+				err := tw.WriteHeader(hdr)
+				if err != nil {
+					return err
+				}
+				addedFiles[root] = true
+			}
+		}
+		addedFiles[f.Path] = true
+		if f.Directory {
+			if f.Contents != nil {
+				return errors.New("Directory with contents not allowed")
+			}
+			hdr := &tar.Header{
+				Name:     f.Path,
+				Typeflag: tar.TypeDir,
+				Mode:     mode,
+			}
+			err := tw.WriteHeader(hdr)
+			if err != nil {
+				return err
+			}
+		} else if f.Symlink != "" {
+			hdr := &tar.Header{
+				Name:     f.Path,
+				Typeflag: tar.TypeSymlink,
+				Mode:     mode,
+				Linkname: f.Symlink,
+			}
+			err := tw.WriteHeader(hdr)
+			if err != nil {
+				return err
+			}
+		} else {
+			hdr := &tar.Header{
+				Name: f.Path,
+				Mode: mode,
+				Size: int64(len(contents)),
+			}
+			err := tw.WriteHeader(hdr)
+			if err != nil {
+				return err
+			}
+			_, err = tw.Write(contents)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}

src/moby/config.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import (
 	"encoding/json"
@@ -24,8 +24,8 @@ type Moby struct {
 		Cmdline string
 	}
 	Init     []string
-	Onboot   []MobyImage
-	Services []MobyImage
+	Onboot   []Image
+	Services []Image
 	Trust    TrustConfig
 	Files    []struct {
 		Path      string
@@ -44,8 +44,8 @@ type TrustConfig struct {
 	Org   []string
 }
 
-// MobyImage is the type of an image config
-type MobyImage struct {
+// Image is the type of an image config
+type Image struct {
 	Name              string             `yaml:"name" json:"name"`
 	Image             string             `yaml:"image" json:"image"`
 	Capabilities      *[]string          `yaml:"capabilities" json:"capabilities,omitempty"`
@@ -153,11 +153,11 @@ func AppendConfig(m0, m1 Moby) Moby {
 	return moby
 }
 
-// NewImage validates an parses yaml or json for a MobyImage
-func NewImage(config []byte) (MobyImage, error) {
+// NewImage validates an parses yaml or json for a Image
+func NewImage(config []byte) (Image, error) {
 	log.Debugf("Reading label config: %s", string(config))
 
-	mi := MobyImage{}
+	mi := Image{}
 
 	// Parse raw yaml
 	var rawYaml interface{}
@@ -220,7 +220,7 @@ func NewImage(config []byte) (MobyImage, error) {
 }
 
 // ConfigToOCI converts a config specification to an OCI config file
-func ConfigToOCI(image MobyImage, trust bool) ([]byte, error) {
+func ConfigToOCI(image Image, trust bool) ([]byte, error) {
 
 	// TODO pass through same docker client to all functions
 	cli, err := dockerClient()
@@ -421,7 +421,7 @@ func assignStringEmpty4(v1, v2, v3, v4 string) string {
 }
 
 // ConfigInspectToOCI converts a config and the output of image inspect to an OCI config
-func ConfigInspectToOCI(yaml MobyImage, inspect types.ImageInspect) (specs.Spec, error) {
+func ConfigInspectToOCI(yaml Image, inspect types.ImageInspect) (specs.Spec, error) {
 	oci := specs.Spec{}
 
 	var inspectConfig container.Config
@@ -430,7 +430,7 @@ func ConfigInspectToOCI(yaml MobyImage, inspect types.ImageInspect) (specs.Spec,
 	}
 
 	// look for org.mobyproject.config label
-	var label MobyImage
+	var label Image
 	labelString := inspectConfig.Labels["org.mobyproject.config"]
 	if labelString != "" {
 		var err error

src/moby/config_test.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import (
 	"encoding/json"
@@ -12,7 +12,7 @@ import (
 func TestOverrides(t *testing.T) {
 	var yamlCaps = []string{"CAP_SYS_ADMIN"}
 
-	var yaml = MobyImage{
+	var yaml = Image{
 		Name:         "test",
 		Image:        "testimage",
 		Capabilities: &yamlCaps,
@@ -20,7 +20,7 @@ func TestOverrides(t *testing.T) {
 
 	var labelCaps = []string{"CAP_SYS_CHROOT"}
 
-	var label = MobyImage{
+	var label = Image{
 		Capabilities: &labelCaps,
 		Cwd:          "/label/directory",
 	}

src/moby/docker.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 // We want to replace much of this with use of containerd tools
 // and also using the Docker API not shelling out

src/moby/image.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import (
 	"archive/tar"

src/moby/linuxkit.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import (
 	"bytes"
@@ -60,7 +60,7 @@ func ensureLinuxkitImage(name string) error {
 	}
 	// TODO pass through --pull to here
 	buf := new(bytes.Buffer)
-	buildInternal(m, buf, false, nil)
+	Build(m, buf, false, "")
 	image := buf.Bytes()
 	kernel, initrd, cmdline, err := tarToInitrd(image)
 	if err != nil {

src/moby/output.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import (
 	"archive/tar"
@@ -128,7 +128,8 @@ func ensurePrereq(out string) error {
 	return err
 }
 
-func validateOutputs(out outputList) error {
+// ValidateOutputs checks if the output type is known
+func ValidateOutputs(out []string) error {
 	log.Debugf("validating output: %v", out)
 
 	for _, o := range out {
@@ -145,10 +146,11 @@ func validateOutputs(out outputList) error {
 	return nil
 }
 
-func outputs(base string, image []byte, out outputList, size int, hyperkit bool) error {
+// Outputs generates all the specified output formats
+func Outputs(base string, image []byte, out []string, size int, hyperkit bool) error {
 	log.Debugf("output: %v %s", out, base)
 
-	err := validateOutputs(out)
+	err := ValidateOutputs(out)
 	if err != nil {
 		return err
 	}

src/moby/schema.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 var schema = string(`
 {

src/moby/trust.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import (
 	"crypto/tls"

src/moby/trust_test.go

@@ -1,4 +1,4 @@
-package main
+package moby
 
 import "testing"
 

src/moby/util.go

@@ -0,0 +1,6 @@
+package moby
+
+var (
+	// MobyDir is the location of the cache directory which should be set by the caller
+	MobyDir string
+)

src/moby/util_unix.go

@@ -0,0 +1,11 @@
+// +build !windows
+
+package moby
+
+import (
+	"os"
+)
+
+func homeDir() string {
+	return os.Getenv("HOME")
+}

src/moby/util_windows.go

@@ -0,0 +1,9 @@
+package moby
+
+import (
+	"os"
+)
+
+func homeDir() string {
+	return os.Getenv("USERPROFILE")
+}