Merge pull request #22 from justincormack/tar-boot-rework

Add tar output format

cmd/moby/build.go

@@ -13,7 +13,6 @@ import (
 	"strings"
 
 	log "github.com/Sirupsen/logrus"
-	"github.com/linuxkit/linuxkit/src/initrd"
 )
 
 const defaultNameForStdin = "moby"
@@ -64,13 +63,38 @@ func build(args []string) {
 		}
 	}
 
-	buildInternal(name, *buildPull, config)
+	m, err := NewConfig(config)
+	if err != nil {
+		log.Fatalf("Invalid config: %v", err)
 	}
 
-func initrdAppend(iw *initrd.Writer, r io.Reader) {
+	image := buildInternal(m, name, *buildPull)
-	_, err := initrd.Copy(iw, r)
+
+	log.Infof("Create outputs:")
+	err = outputs(m, name, image)
 	if err != nil {
-		log.Fatalf("initrd write error: %v", err)
+		log.Fatalf("Error writing outputs: %v", err)
+	}
+}
+
+func initrdAppend(iw *tar.Writer, r io.Reader) {
+	tr := tar.NewReader(r)
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatalln(err)
+		}
+		err = iw.WriteHeader(hdr)
+		if err != nil {
+			log.Fatalln(err)
+		}
+		_, err = io.Copy(iw, tr)
+		if err != nil {
+			log.Fatalln(err)
+		}
 	}
 }
 
@@ -100,14 +124,10 @@ func enforceContentTrust(fullImageName string, config *TrustConfig) bool {
 }
 
 // Perform the actual build process
-func buildInternal(name string, pull bool, config []byte) {
+// TODO return error not panic
-	m, err := NewConfig(config)
+func buildInternal(m *Moby, name string, pull bool) []byte {
-	if err != nil {
-		log.Fatalf("Invalid config: %v", err)
-	}
-
 	w := new(bytes.Buffer)
-	iw := initrd.NewWriter(w)
+	iw := tar.NewWriter(w)
 
 	if pull || enforceContentTrust(m.Kernel.Image, &m.Trust) {
 		log.Infof("Pull kernel image: %s", m.Kernel.Image)
@@ -129,10 +149,11 @@ func buildInternal(name string, pull bool, config []byte) {
 	}
 	buf := bytes.NewBuffer(out)
 
-	kernel, ktar, err := untarKernel(buf, kernelName, kernelAltName, ktarName)
+	kernel, ktar, err := untarKernel(buf, kernelName, kernelAltName, ktarName, m.Kernel.Cmdline)
 	if err != nil {
 		log.Fatalf("Could not extract kernel image and filesystem from tarball. %v", err)
 	}
+	initrdAppend(iw, kernel)
 	initrdAppend(iw, ktar)
 
 	// convert init images to tarballs
@@ -191,14 +212,10 @@ func buildInternal(name string, pull bool, config []byte) {
 		log.Fatalf("initrd close error: %v", err)
 	}
 
-	log.Infof("Create outputs:")
+	return w.Bytes()
-	err = outputs(m, name, kernel.Bytes(), w.Bytes())
-	if err != nil {
-		log.Fatalf("Error writing outputs: %v", err)
-	}
 }
 
-func untarKernel(buf *bytes.Buffer, kernelName, kernelAltName, ktarName string) (*bytes.Buffer, *bytes.Buffer, error) {
+func untarKernel(buf *bytes.Buffer, kernelName, kernelAltName, ktarName string, cmdline string) (*bytes.Buffer, *bytes.Buffer, error) {
 	tr := tar.NewReader(buf)
 
 	var kernel, ktar *bytes.Buffer
@@ -219,10 +236,45 @@ func untarKernel(buf *bytes.Buffer, kernelName, kernelAltName, ktarName string)
 			}
 			foundKernel = true
 			kernel = new(bytes.Buffer)
-			_, err := io.Copy(kernel, tr)
+			// make a new tarball with kernel in /boot/kernel
+			tw := tar.NewWriter(kernel)
+			whdr := &tar.Header{
+				Name:     "boot",
+				Mode:     0700,
+				Typeflag: tar.TypeDir,
+			}
+			if err := tw.WriteHeader(whdr); err != nil {
+				return nil, nil, err
+			}
+			whdr = &tar.Header{
+				Name: "boot/kernel",
+				Mode: hdr.Mode,
+				Size: hdr.Size,
+			}
+			if err := tw.WriteHeader(whdr); err != nil {
+				return nil, nil, err
+			}
+			_, err = io.Copy(tw, tr)
 			if err != nil {
 				return nil, nil, err
 			}
+			// add the cmdline in /boot/cmdline
+			whdr = &tar.Header{
+				Name: "boot/cmdline",
+				Mode: 0700,
+				Size: int64(len(cmdline)),
+			}
+			if err := tw.WriteHeader(whdr); err != nil {
+				return nil, nil, err
+			}
+			buf := bytes.NewBufferString(cmdline)
+			_, err = io.Copy(tw, buf)
+			if err != nil {
+				return nil, nil, err
+			}
+			if err := tw.Close(); err != nil {
+				return nil, nil, err
+			}
 		case ktarName:
 			ktar = new(bytes.Buffer)
 			_, err := io.Copy(ktar, tr)

cmd/moby/output.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 
+	"github.com/linuxkit/linuxkit/src/initrd"
 	log "github.com/Sirupsen/logrus"
 )
 
@@ -19,42 +20,76 @@ const (
 	vmdk = "linuxkit/mkimage-vmdk:182b541474ca7965c8e8f987389b651859f760da@sha256:99638c5ddb17614f54c6b8e11bd9d49d1dea9d837f38e0f6c1a5f451085d449b"
 )
 
-func outputs(m *Moby, base string, kernel []byte, initrd []byte) error {
+func outputs(m *Moby, base string, image []byte) error {
 	log.Debugf("output: %s %s", m.Outputs, base)
+
 	for _, o := range m.Outputs {
 		switch o.Format {
+		case "tar":
+			err := outputTar(base, image)
+			if err != nil {
+				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
+			}
 		case "kernel+initrd":
-			err := outputKernelInitrd(base, kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+			if err != nil {
+				return fmt.Errorf("Error converting to initrd: %v", err)
+			}
+			err = outputKernelInitrd(base, kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
 		case "iso-bios":
-			err := outputISO(bios, base+".iso", kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+                        if err != nil {
+                                return fmt.Errorf("Error converting to initrd: %v", err)
+                        }
+			err = outputISO(bios, base+".iso", kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
 		case "iso-efi":
-			err := outputISO(efi, base+"-efi.iso", kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+                        if err != nil {
+                                return fmt.Errorf("Error converting to initrd: %v", err)
+                        }
+			err = outputISO(efi, base+"-efi.iso", kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
 		case "gcp-img":
-			err := outputImg(gcp, base+".img.tar.gz", kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+                        if err != nil {
+                                return fmt.Errorf("Error converting to initrd: %v", err)
+                        }
+			err = outputImg(gcp, base+".img.tar.gz", kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
 		case "qcow", "qcow2":
-			err := outputImg(qcow, base+".qcow2", kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+                        if err != nil {
+                                return fmt.Errorf("Error converting to initrd: %v", err)
+                        }
+			err = outputImg(qcow, base+".qcow2", kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
 		case "vhd":
-			err := outputImg(vhd, base+".vhd", kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+                        if err != nil {
+                                return fmt.Errorf("Error converting to initrd: %v", err)
+                        }
+			err = outputImg(vhd, base+".vhd", kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
 		case "vmdk":
-			err := outputImg(vmdk, base+".vmdk", kernel, initrd, m.Kernel.Cmdline)
+			kernel, initrd, cmdline, err := tarToInitrd(image)
+                        if err != nil {
+                                return fmt.Errorf("Error converting to initrd: %v", err)
+                        }
+			err = outputImg(vmdk, base+".vmdk", kernel, initrd, cmdline)
 			if err != nil {
 				return fmt.Errorf("Error writing %s output: %v", o.Format, err)
 			}
@@ -67,6 +102,18 @@ func outputs(m *Moby, base string, kernel []byte, initrd []byte) error {
 	return nil
 }
 
+func tarToInitrd(image []byte) ([]byte, []byte, string, error) {
+	w := new(bytes.Buffer)
+	iw := initrd.NewWriter(w)
+	r := bytes.NewReader(image)
+	tr := tar.NewReader(r)
+	kernel, cmdline, err := initrd.CopySplitTar(iw, tr)
+	if err != nil {
+		return []byte{}, []byte{}, "", err
+	}
+	return kernel, w.Bytes(), cmdline, nil
+}
+
 func tarInitrdKernel(kernel, initrd []byte) (*bytes.Buffer, error) {
 	buf := new(bytes.Buffer)
 	tw := tar.NewWriter(buf)
@@ -156,3 +203,9 @@ func outputKernelInitrd(base string, kernel []byte, initrd []byte, cmdline strin
 	}
 	return nil
 }
+
+func outputTar(base string, initrd []byte) error {
+	log.Debugf("output tar: %s", base)
+	log.Infof("  %s", base+".tar")
+	return ioutil.WriteFile(base+".tar", initrd, os.FileMode(0644))
+}

vendor.conf

@@ -4,7 +4,7 @@ github.com/docker/docker 420b67f892d5424be59a788a51e2c4e64bb9cd66
 github.com/docker/go-connections e15c02316c12de00874640cd76311849de2aeed5
 github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
 github.com/docker/libtrust 9cbd2a1374f46905c68a4eb3694a130610adc62a
-github.com/linuxkit/linuxkit 22514db912b9bec73e2a3c3ffd1611633c25b9db
+github.com/linuxkit/linuxkit 17dd50cec61de35c48b786998a154678dc46ff6a
 github.com/opencontainers/go-digest a6d0ee40d4207ea02364bd3b9e8e77b9159ba1eb
 github.com/opencontainers/runtime-spec d094a5c9c1997ab086197b57e9378fabed394d92
 github.com/pkg/errors ff09b135c25aae272398c51a07235b90a75aa4f0

vendor/github.com/linuxkit/linuxkit/README.md

@@ -22,10 +22,8 @@ LinuxKit uses the `moby` tool for image builds, and the `linuxkit` tool for push
 Simple build instructions: use `make` to build. This will build the tools in `bin/`. Add this
 to your `PATH` or copy it to somewhere in your `PATH` eg `sudo cp bin/* /usr/local/bin/`. Or you can use `sudo make install`.
 
-If you already have `go` installed you can use `go get -u github.com/linuxkit/linuxkit/src/cmd/moby` to install
+If you already have `go` installed you can use `go get -u github.com/moby/tool/cmd/moby` to install
 the `moby` build tool, and `go get -u github.com/linuxkit/linuxkit/src/cmd/linuxkit` to install the `linuxkit` tool.
-You can use `go get -u github.com/linuxkit/linuxkit/src/cmd/infrakit-instance-hyperkit`
-to get the hyperkit infrakit tool.
 
 Once you have built the tool, use `moby build linuxkit.yml` to build the example configuration,
 and `linuxkit run linuxkit` to run locally. Use `halt` to terminate on the console.
@@ -43,10 +41,11 @@ See `linuxkit run --help`.
 
 `make test` or `make test-hyperkit` will run the test suite
 
-There are also docs for booting on [Google Cloud](docs/gcp.md); `linuxkit push gcp <name> && linuxkit run gcp <name>.yml` should
+Additional, platform specific information is available for:
-work if you specified a GCP image to be built in the config.
+- [macOS](docs/mac.md)
+- [Google Cloud](docs/gcp.md)
 
-More detailed docs will be available shortly, for running both single hosts and clusters.
+We'll add more detailed docs for other platforms in the future.
 
 ## Building your own customised image
 

vendor/github.com/linuxkit/linuxkit/projects/README.md

@@ -20,6 +20,7 @@ If you want to create a project, please submit a pull request to create a new di
 - [Landlock LSM](landlock/) programmatic access control
 - [Clear Containers](clear-containers/) Clear Containers image
 - [Logging](logging/) Experimental logging tools
+- [etcd cluster](etcd/) etcd cluster demo from DockerCon'17
 
 ## Current projects not yet documented
 - VMWare support (VMWare)

vendor/github.com/linuxkit/linuxkit/src/initrd/initrd.go

@@ -6,6 +6,7 @@ import (
 	"compress/gzip"
 	"errors"
 	"io"
+	"io/ioutil"
 
 	"github.com/linuxkit/linuxkit/src/pad4"
 	"github.com/surma/gocpio"
@@ -92,6 +93,68 @@ func CopyTar(w *Writer, r *tar.Reader) (written int64, err error) {
 	}
 }
 
+// CopySplitTar copies a tar stream into an initrd, but splits out kernel and cmdline
+func CopySplitTar(w *Writer, r *tar.Reader) (kernel []byte, cmdline string, err error) {
+	for {
+		var thdr *tar.Header
+		thdr, err = r.Next()
+		if err == io.EOF {
+			return kernel, cmdline, nil
+		}
+		if err != nil {
+			return
+		}
+		tp := typeconv(thdr)
+		if tp == -1 {
+			return kernel, cmdline, errors.New("cannot convert tar file")
+		}
+		switch thdr.Name {
+		case "boot/kernel":
+			kernel, err = ioutil.ReadAll(r)
+			if err != nil {
+				return
+			}
+		case "boot/cmdline":
+			var buf []byte
+			buf, err = ioutil.ReadAll(r)
+			if err != nil {
+				return
+			}
+			cmdline = string(buf)
+		case "boot":
+		default:
+			size := thdr.Size
+			if tp == cpio.TYPE_SYMLINK {
+				size = int64(len(thdr.Linkname))
+			}
+			chdr := cpio.Header{
+				Mode:     thdr.Mode,
+				Uid:      thdr.Uid,
+				Gid:      thdr.Gid,
+				Mtime:    thdr.ModTime.Unix(),
+				Size:     size,
+				Devmajor: thdr.Devmajor,
+				Devminor: thdr.Devminor,
+				Type:     tp,
+				Name:     thdr.Name,
+			}
+			err = w.WriteHeader(&chdr)
+			if err != nil {
+				return
+			}
+			if tp == cpio.TYPE_SYMLINK {
+				buffer := bytes.NewBufferString(thdr.Linkname)
+				_, err = io.Copy(w, buffer)
+			} else {
+				_, err = io.Copy(w, r)
+			}
+			if err != nil {
+				return
+			}
+		}
+	}
+}
+
 // NewWriter creates a writer that will output an initrd stream
 func NewWriter(w io.Writer) *Writer {
 	initrd := new(Writer)

vendor/github.com/linuxkit/linuxkit/vendor.conf

@@ -1,24 +1,12 @@
-github.com/google/google-api-go-client 16ab375f94503bfa0d19db78e96bffbe1a34354f
-github.com/Masterminds/semver 312afcd0e81e5cf81fdc3cfd0e8504ae031521c8
-github.com/Masterminds/sprig 01a849f546a584d7b29bfee253e7db0aed44f7ba
 github.com/Sirupsen/logrus 10f801ebc38b33738c9d17d50860f484a0988ff5
-github.com/aokoli/goutils 9c37978a95bd5c709a15883b6242714ea6709e64
-github.com/armon/go-radix 4239b77079c7b5d1243b7b4736304ce8ddb6f0f2
 github.com/docker/docker 8d96619e5a367798cffcb740cfc41e0a505a5232
 github.com/docker/distribution 07f32ac1831ed0fc71960b7da5d6bb83cb6881b5
 github.com/docker/engine-api cf82c64276ebc2501e72b241f9fdc1e21e421743
 github.com/docker/go-connections e15c02316c12de00874640cd76311849de2aeed5
 github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
 github.com/docker/infrakit cb420e3e50ea60afe58538b1d3cab1cb14059433
-github.com/ghodss/yaml 0ca9ea5df5451ffdf184b4428c902747c2c11cd7
 github.com/golang/protobuf c9c7427a2a70d2eb3bafa0ab2dc163e45f143317
 github.com/googleapis/gax-go 8c5154c0fe5bf18cf649634d4c6df50897a32751
-github.com/gorilla/context 08b5f424b9271eedf6f9f0ce86cb9396ed337a42
-github.com/gorilla/mux 599cba5e7b6137d46ddf58fb1765f5d928e69604
-github.com/gorilla/rpc 22c016f3df3febe0c1f6727598b6389507e03a18
-github.com/inconshreveable/mousetrap 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75
-github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
-github.com/mattn/go-colorable d228849
 github.com/mitchellh/go-ps 4fdf99ab29366514c69ccccddab5dc58b8d84062
 github.com/moby/hyperkit 9b5f5fd848f0f5aedccb67a5a8cfa6787b8654f9
 github.com/opencontainers/runtime-spec d094a5c9c1997ab086197b57e9378fabed394d92
@@ -26,11 +14,7 @@ github.com/pkg/errors ff09b135c25aae272398c51a07235b90a75aa4f0
 github.com/packethost/packngo 91d54000aa56874149d348a884ba083c41d38091
 github.com/rneugeba/iso9660wrap 4606f848a055435cdef85305960b0e1bb788d506
 github.com/satori/go.uuid b061729afc07e77a8aa4fad0a2fd840958f1942a
-github.com/spf13/afero 9be650865eab0c12963d8753212f4f9c66cdcf12
-github.com/spf13/cobra 7be4beda01ec05d0b93d80b3facd2b6f44080d94
-github.com/spf13/pflag 9ff6c6923cfffbcd502984b8e0c80539a94968b7
 github.com/surma/gocpio fcb68777e7dc4ea43ffce871b552c0d073c17495
-github.com/vaughan0/go-ini a98ad7ee00ec53921f08832bc06ecf7fd600e6a1
 github.com/xeipuuv/gojsonpointer 6fe8760cad3569743d51ddbb243b26f8456742dc
 github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45
 github.com/xeipuuv/gojsonschema 702b404897d4364af44dc8dcabc9815947942325
@@ -38,9 +22,6 @@ golang.org/x/crypto 573951cbe80bb6352881271bb276f48749eab6f4
 golang.org/x/net a6577fac2d73be281a500b310739095313165611
 golang.org/x/oauth2 1611bb46e67abc64a71ecc5c3ae67f1cbbc2b921
 golang.org/x/sys 99f16d856c9836c42d24e7ab64ea72916925fa97
-golang.org/x/text a263ba8
 google.golang.org/api 1202890e803f07684581b575fda809bf335a533f
 google.golang.org/grpc 0713829b980f4ddd276689a36235c5fcc82a21bf
-gopkg.in/inconshreveable/log15.v2 v2.11
-gopkg.in/tylerb/graceful.v1 4654dfbb6ad53cb5e27f37d99b02e16c1872fbbb
 gopkg.in/yaml.v2 a3f3340b5840cee44f372bddb5880fcbc419b46a