Merge pull request #49 from justincormack/linuxkit

Use linuxkit to build qcow2 and raw image disks, rather than docker containers with libguestfs

cmd/moby/build.go

@@ -11,6 +11,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strconv"
 	"strings"
 
 	log "github.com/Sirupsen/logrus"
@@ -50,8 +51,10 @@ func build(args []string) {
 	}
 	buildName := buildCmd.String("name", "", "Name to use for output files")
 	buildDir := buildCmd.String("dir", "", "Directory for output files, default current directory")
+	buildSize := buildCmd.String("size", "1024M", "Size for output image, if supported and fixed size")
 	buildPull := buildCmd.Bool("pull", false, "Always pull images")
 	buildDisableTrust := buildCmd.Bool("disable-content-trust", false, "Skip image trust verification specified in trust section of config (default false)")
+	buildHyperkit := buildCmd.Bool("hyperkit", false, "Use hyperkit for LinuxKit based builds where possible")
 	buildCmd.Var(&buildOut, "output", "Output types to create [ "+strings.Join(outputTypes, " ")+" ]")
 
 	if err := buildCmd.Parse(args); err != nil {
@@ -59,6 +62,12 @@ func build(args []string) {
 	}
 	remArgs := buildCmd.Args()
 
+	if len(remArgs) == 0 {
+		fmt.Println("Please specify a configuration file")
+		buildCmd.Usage()
+		os.Exit(1)
+	}
+
 	if len(buildOut) == 0 {
 		buildOut = outputList{"kernel+initrd"}
 	}
@@ -72,11 +81,11 @@ func build(args []string) {
 		os.Exit(1)
 	}
 
-	if len(remArgs) == 0 {
+	size, err := getDiskSizeMB(*buildSize)
-		fmt.Println("Please specify a configuration file")
+	if err != nil {
-		buildCmd.Usage()
+		log.Fatalf("Unable to parse disk size: %v", err)
-		os.Exit(1)
 	}
+
 	name := *buildName
 	var config []byte
 	if conf := remArgs[0]; conf == "-" {
@@ -115,12 +124,33 @@ func build(args []string) {
 	image := buildInternal(m, *buildPull)
 
 	log.Infof("Create outputs:")
-	err = outputs(filepath.Join(*buildDir, name), image, buildOut)
+	err = outputs(filepath.Join(*buildDir, name), image, buildOut, size, *buildHyperkit)
 	if err != nil {
 		log.Fatalf("Error writing outputs: %v", err)
 	}
 }
 
+// Parse a string which is either a number in MB, or a number with
+// either M (for Megabytes) or G (for GigaBytes) as a suffix and
+// returns the number in MB. Return 0 if string is empty.
+func getDiskSizeMB(s string) (int, error) {
+	if s == "" {
+		return 0, nil
+	}
+	sz := len(s)
+	if strings.HasSuffix(s, "G") {
+		i, err := strconv.Atoi(s[:sz-1])
+		if err != nil {
+			return 0, err
+		}
+		return i * 1024, nil
+	}
+	if strings.HasSuffix(s, "M") {
+		s = s[:sz-1]
+	}
+	return strconv.Atoi(s)
+}
+
 func initrdAppend(iw *tar.Writer, r io.Reader) {
 	tr := tar.NewReader(r)
 	for {

cmd/moby/linuxkit.go

@@ -0,0 +1,140 @@
+package main
+
+import (
+	"crypto/sha256"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	log "github.com/Sirupsen/logrus"
+)
+
+var linuxkitYaml = map[string]string{"mkimage": `
+kernel:
+  image: "linuxkit/kernel:4.9.x"
+  cmdline: "console=ttyS0"
+init:
+  - linuxkit/init:1b8a7e394d2ec2f1fdb4d67645829d1b5bdca037
+  - linuxkit/runc:3a4e6cbf15470f62501b019b55e1caac5ee7689f
+  - linuxkit/containerd:b1766e4c4c09f63ac4925a6e4612852a93f7e73b
+onboot:
+  - name: mkimage
+    image: "linuxkit/mkimage:f4bf0c24261f7d120c8674892805ab3054eb8ac3"
+  - name: poweroff
+    image: "linuxkit/poweroff:a8f1e4ad8d459f1fdaad9e4b007512cb3b504ae8"
+trust:
+  org:
+    - linuxkit
+`}
+
+func imageFilename(name string) string {
+	yaml := linuxkitYaml[name]
+	hash := sha256.Sum256([]byte(yaml))
+	return filepath.Join(MobyDir, "linuxkit", name+"-"+fmt.Sprintf("%x", hash))
+}
+
+func ensureLinuxkitImage(name string) error {
+	filename := imageFilename(name)
+	_, err1 := os.Stat(filename + "-kernel")
+	_, err2 := os.Stat(filename + "-initrd.img")
+	_, err3 := os.Stat(filename + "-cmdline")
+	if err1 == nil && err2 == nil && err3 == nil {
+		return nil
+	}
+	err := os.MkdirAll(filepath.Join(MobyDir, "linuxkit"), 0755)
+	if err != nil {
+		return err
+	}
+	// TODO clean up old files
+	log.Infof("Building LinuxKit image %s to generate output formats", name)
+
+	yaml := linuxkitYaml[name]
+
+	m, err := NewConfig([]byte(yaml))
+	if err != nil {
+		return err
+	}
+	// TODO pass through --pull to here
+	image := buildInternal(m, false)
+	kernel, initrd, cmdline, err := tarToInitrd(image)
+	if err != nil {
+		return fmt.Errorf("Error converting to initrd: %v", err)
+	}
+	err = writeKernelInitrd(filename, kernel, initrd, cmdline)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func writeKernelInitrd(filename string, kernel []byte, initrd []byte, cmdline string) error {
+	err := ioutil.WriteFile(filename+"-kernel", kernel, 0600)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(filename+"-initrd.img", initrd, 0600)
+	if err != nil {
+		return err
+	}
+	err = ioutil.WriteFile(filename+"-cmdline", []byte(cmdline), 0600)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func outputLinuxKit(format string, filename string, kernel []byte, initrd []byte, cmdline string, size int, hyperkit bool) error {
+	log.Debugf("output linuxkit generated img: %s %s size %d", format, filename, size)
+
+	tmp, err := ioutil.TempDir("", "moby")
+	if err != nil {
+		return err
+	}
+	defer os.RemoveAll(tmp)
+
+	buf, err := tarInitrdKernel(kernel, initrd, cmdline)
+	if err != nil {
+		return err
+	}
+
+	tardisk := filepath.Join(tmp, "tardisk")
+	f, err := os.Create(tardisk)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(f, buf)
+	if err != nil {
+		return err
+	}
+	err = f.Close()
+	if err != nil {
+		return err
+	}
+
+	sizeString := fmt.Sprintf("%dM", size)
+	_ = os.Remove(filename)
+	_, err = os.Stat(filename)
+	if err == nil || !os.IsNotExist(err) {
+		return fmt.Errorf("Cannot remove existing file [%s]", filename)
+	}
+	linuxkit, err := exec.LookPath("linuxkit")
+	if err != nil {
+		return fmt.Errorf("Cannot find linuxkit executable, needed to build %s output type: %v", format, err)
+	}
+	commandLine := []string{"-q", "run", "qemu", "-disk", fmt.Sprintf("%s,size=%s,format=%s", filename, sizeString, format), "-disk", fmt.Sprintf("%s,format=raw", tardisk), "-kernel", imageFilename("mkimage")}
+	// if hyperkit && format == "raw" {
+	// TODO support hyperkit
+	// }
+	log.Debugf("run %s: %v", linuxkit, commandLine)
+	cmd := exec.Command(linuxkit, commandLine...)
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}

cmd/moby/output.go

@@ -3,9 +3,12 @@ package main
 import (
 	"archive/tar"
 	"bytes"
+	"compress/gzip"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/linuxkit/linuxkit/src/initrd"
@@ -14,22 +17,19 @@ import (
 const (
 	bios = "linuxkit/mkimage-iso-bios:db791abed6f2b5320feb6cec255a635aee3756f6@sha256:e57483075307bcea4a7257f87eee733d3e24e7a964ba15dcc01111df6729ab3b"
 	efi  = "linuxkit/mkimage-iso-efi:5c2fc616bde288476a14f4f6dd0d273a66832822@sha256:876ef47ec2b30af40e70f1e98f496206eb430915867c4f9f400e1af47fd58d7c"
-	gcp  = "linuxkit/mkimage-gcp:46716b3d3f7aa1a7607a3426fe0ccebc554b14ee@sha256:18d8e0482f65a2481f5b6ba1e7ce77723b246bf13bdb612be5e64df90297940c"
-	img  = "linuxkit/mkimage-img-gz:eb85aac97f716ad8b8e7e593de3378e740ef2eeb@sha256:f1fb2368765a8ba6d1edfb073565550ae98486fb4943fbeb7d05357e5ba9969d"
-	qcow = "linuxkit/mkimage-qcow:69890f35b55e4ff8a2c7a714907f988e57056d02@sha256:f89dc09f82bdbf86d7edae89604544f20b99d99c9b5cabcf1f93308095d8c244"
 	vhd  = "linuxkit/mkimage-vhd:a04c8480d41ca9cef6b7710bd45a592220c3acb2@sha256:ba373dc8ae5dc72685dbe4b872d8f588bc68b2114abd8bdc6a74d82a2b62cce3"
 	vmdk = "linuxkit/mkimage-vmdk:182b541474ca7965c8e8f987389b651859f760da@sha256:99638c5ddb17614f54c6b8e11bd9d49d1dea9d837f38e0f6c1a5f451085d449b"
 )
 
-var outFuns = map[string]func(string, []byte) error{
+var outFuns = map[string]func(string, []byte, int, bool) error{
-	"tar": func(base string, image []byte) error {
+	"tar": func(base string, image []byte, size int, hyperkit bool) error {
 		err := outputTar(base, image)
 		if err != nil {
 			return fmt.Errorf("Error writing tar output: %v", err)
 		}
 		return nil
 	},
-	"kernel+initrd": func(base string, image []byte) error {
+	"kernel+initrd": func(base string, image []byte, size int, hyperkit bool) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -40,7 +40,7 @@ var outFuns = map[string]func(string, []byte) error{
 		}
 		return nil
 	},
-	"iso-bios": func(base string, image []byte) error {
+	"iso-bios": func(base string, image []byte, size int, hyperkit bool) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -51,7 +51,7 @@ var outFuns = map[string]func(string, []byte) error{
 		}
 		return nil
 	},
-	"iso-efi": func(base string, image []byte) error {
+	"iso-efi": func(base string, image []byte, size int, hyperkit bool) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -62,40 +62,137 @@ var outFuns = map[string]func(string, []byte) error{
 		}
 		return nil
 	},
-	"img-gz": func(base string, image []byte) error {
+	"img": func(base string, image []byte, size int, hyperkit bool) error {
+		filename := base + ".img"
+		log.Infof("  %s", filename)
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImgSize(img, base+".img.gz", kernel, initrd, cmdline, "1G")
+		err = outputLinuxKit("raw", filename, kernel, initrd, cmdline, size, hyperkit)
-		if err != nil {
-			return fmt.Errorf("Error writing img-gz output: %v", err)
-		}
-		return nil
-	},
-	"gcp-img": func(base string, image []byte) error {
-		kernel, initrd, cmdline, err := tarToInitrd(image)
-		if err != nil {
-			return fmt.Errorf("Error converting to initrd: %v", err)
-		}
-		err = outputImg(gcp, base+".img.tar.gz", kernel, initrd, cmdline)
-		if err != nil {
-			return fmt.Errorf("Error writing gcp-img output: %v", err)
-		}
-		return nil
-	},
-	"qcow2": func(base string, image []byte) error {
-		kernel, initrd, cmdline, err := tarToInitrd(image)
-		if err != nil {
-			return fmt.Errorf("Error converting to initrd: %v", err)
-		}
-		err = outputImg(qcow, base+".qcow2", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing qcow2 output: %v", err)
 		}
 		return nil
 	},
-	"vhd": func(base string, image []byte) error {
+	"img-gz": func(base string, image []byte, size int, hyperkit bool) error {
+		filename := base + ".img.gz"
+		log.Infof("  %s", filename)
+		kernel, initrd, cmdline, err := tarToInitrd(image)
+		if err != nil {
+			return fmt.Errorf("Error converting to initrd: %v", err)
+		}
+		tmp, err := ioutil.TempDir("", "img-gz")
+		if err != nil {
+			return err
+		}
+		err = outputLinuxKit("raw", filepath.Join(tmp, "uncompressed.img"), kernel, initrd, cmdline, size, hyperkit)
+		if err != nil {
+			return fmt.Errorf("Error writing img-gz output: %v", err)
+		}
+		out, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+		in, err := os.Open(filepath.Join(tmp, "uncompressed.img"))
+		if err != nil {
+			return err
+		}
+		zw := gzip.NewWriter(out)
+		io.Copy(zw, in)
+		err = zw.Close()
+		if err != nil {
+			return err
+		}
+		err = in.Close()
+		if err != nil {
+			return err
+		}
+		err = out.Close()
+		if err != nil {
+			return err
+		}
+		err = os.RemoveAll(tmp)
+		if err != nil {
+			return err
+		}
+		return nil
+	},
+	"gcp-img": func(base string, image []byte, size int, hyperkit bool) error {
+		filename := base + ".img.tar.gz"
+		log.Infof("  %s", filename)
+		kernel, initrd, cmdline, err := tarToInitrd(image)
+		if err != nil {
+			return fmt.Errorf("Error converting to initrd: %v", err)
+		}
+		tmp, err := ioutil.TempDir("", "gcp-img")
+		if err != nil {
+			return err
+		}
+		err = outputLinuxKit("raw", filepath.Join(tmp, "disk.raw"), kernel, initrd, cmdline, size, hyperkit)
+		if err != nil {
+			return fmt.Errorf("Error writing gcp-img output: %v", err)
+		}
+		out, err := os.Create(filename)
+		if err != nil {
+			return err
+		}
+		in, err := os.Open(filepath.Join(tmp, "disk.raw"))
+		if err != nil {
+			return err
+		}
+		fi, err := in.Stat()
+		if err != nil {
+			return err
+		}
+		zw := gzip.NewWriter(out)
+		tw := tar.NewWriter(zw)
+		hdr := &tar.Header{
+			Name: "disk.raw",
+			Mode: 0600,
+			Size: fi.Size(),
+		}
+		err = tw.WriteHeader(hdr)
+		if err != nil {
+			return err
+		}
+		io.Copy(tw, in)
+		err = tw.Close()
+		if err != nil {
+			return err
+		}
+		err = zw.Close()
+		if err != nil {
+			return err
+		}
+		err = in.Close()
+		if err != nil {
+			return err
+		}
+		err = out.Close()
+		if err != nil {
+			return err
+		}
+		err = os.RemoveAll(tmp)
+		if err != nil {
+			return err
+		}
+		return nil
+	},
+	"qcow2": func(base string, image []byte, size int, hyperkit bool) error {
+		filename := base + ".qcow2"
+		log.Infof("  %s", filename)
+		kernel, initrd, cmdline, err := tarToInitrd(image)
+		if err != nil {
+			return fmt.Errorf("Error converting to initrd: %v", err)
+		}
+		err = outputLinuxKit("qcow2", filename, kernel, initrd, cmdline, size, hyperkit)
+		if err != nil {
+			return fmt.Errorf("Error writing qcow2 output: %v", err)
+		}
+		return nil
+	},
+	"vhd": func(base string, image []byte, size int, hyperkit bool) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -106,7 +203,7 @@ var outFuns = map[string]func(string, []byte) error{
 		}
 		return nil
 	},
-	"vmdk": func(base string, image []byte) error {
+	"vmdk": func(base string, image []byte, size int, hyperkit bool) error {
 		kernel, initrd, cmdline, err := tarToInitrd(image)
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
@@ -119,6 +216,22 @@ var outFuns = map[string]func(string, []byte) error{
 	},
 }
 
+var prereq = map[string]string{
+	"img":     "mkimage",
+	"img-gz":  "mkimage",
+	"gcp-img": "mkimage",
+	"qcow2":   "mkimage",
+}
+
+func ensurePrereq(out string) error {
+	var err error
+	p := prereq[out]
+	if p != "" {
+		err = ensureLinuxkitImage(p)
+	}
+	return err
+}
+
 func validateOutputs(out outputList) error {
 	log.Debugf("validating output: %v", out)
 
@@ -127,12 +240,16 @@ func validateOutputs(out outputList) error {
 		if f == nil {
 			return fmt.Errorf("Unknown output type %s", o)
 		}
+		err := ensurePrereq(o)
+		if err != nil {
+			return fmt.Errorf("Failed to set up output type %s: %v", o, err)
+		}
 	}
 
 	return nil
 }
 
-func outputs(base string, image []byte, out outputList) error {
+func outputs(base string, image []byte, out outputList, size int, hyperkit bool) error {
 	log.Debugf("output: %v %s", out, base)
 
 	err := validateOutputs(out)
@@ -141,7 +258,7 @@ func outputs(base string, image []byte, out outputList) error {
 	}
 	for _, o := range out {
 		f := outFuns[o]
-		err = f(base, image)
+		err := f(base, image, size, hyperkit)
 		if err != nil {
 			return err
 		}
@@ -230,19 +347,19 @@ func outputImg(image, filename string, kernel []byte, initrd []byte, cmdline str
 	return nil
 }
 
-// this should replace the other version for types that can specify a size, and get size from CLI in future
+// this should replace the other version for types that can specify a size
-func outputImgSize(image, filename string, kernel []byte, initrd []byte, cmdline string, size string) error {
+func outputImgSize(image, filename string, kernel []byte, initrd []byte, cmdline string, size int) error {
-	log.Debugf("output img: %s %s size %s", image, filename, size)
+	log.Debugf("output img: %s %s size %d", image, filename, size)
 	log.Infof("  %s", filename)
 	buf, err := tarInitrdKernel(kernel, initrd, cmdline)
 	if err != nil {
 		return err
 	}
 	var img []byte
-	if size == "" {
+	if size == 0 {
 		img, err = dockerRunInput(buf, image)
 	} else {
-		img, err = dockerRunInput(buf, image, size)
+		img, err = dockerRunInput(buf, image, fmt.Sprintf("%dM", size))
 	}
 	if err != nil {
 		return err