Merge pull request #3002 from rn/squashfssupport

Support building and running with SquashFS root filesystem

docs/platform-hyperkit.md

@@ -12,9 +12,19 @@ Alternatively, you can install HyperKit and VPNKit standalone and use it without
 
 ## Boot
 
-The HyperKit backend currently supports booting the
+The HyperKit backend currently supports booting:
-`kernel+initrd` output from `moby`, and EFI ISOs using the EFI firmware.
+- `kernel+initrd` output from `linuxkit build`.
+- `kernel+squashfs` output from `linuxkit build`.
+- EFI ISOs using the EFI firmware.
 
+You need to select the boot method manually using the command line
+options. The default is `kernel+initrd`. `kernel+squashfs` can be
+selected using `-squashfs` and to boot a ISO with EFI you have to
+specify `-iso -uefi`.
+
+The `kernel+initrd` uses a RAM disk for the root filesystem. If you
+have RAM constraints or large images we recommend using either the
+`kernel+squashfs` or the EFI ISO boot.
 
 ## Console
 

docs/platform-qemu.md

@@ -24,6 +24,7 @@ specified with `-arch` and currently accepts `x86_64`, `aarch64`, and
 `linuxkit run qemu` can boot in different types of images:
 
 - `kernel+initrd`: This is the default mode of `linuxkit run qemu` [`x86_64`, `arm64`, `s390x`]
+- `kernel+squashfs`: `linuxkit run qemu -squashfs <path to directory>`. This expects a kernel and a squashfs image. [`x86_64`, `arm64`, `s390x`]
 - `iso-bios`: `linuxkit run qemu -iso <path to iso>` [`x86_64`]
 - `iso-efi`: `linuxkit run qemu -iso -uefi <path to iso>`. This looks in `/usr/share/ovmf/bios.bin` for the EFI firmware by default. Can be overwritten with `-fw`. [`x86_64`, `arm64`]
 - `qcow-bios`: `linuxkit run qemu disk.qcow2` [`x86_64`]
@@ -32,6 +33,10 @@ specified with `-arch` and currently accepts `x86_64`, `aarch64`, and
 
 The formats `qcow-efi` and `raw-efi` may also work, but are currently not tested.
 
+The default `kernel+initrd` boot uses a RAM disk for the root
+filesystem. If you have RAM constraints or large images we recommend
+using one of the other methods, such as `kernel+squashfs` or booting
+via a ISO image.
 
 ## Console
 

src/cmd/linuxkit/main.go

@@ -8,11 +8,34 @@ import (
 	"path/filepath"
 
 	"github.com/linuxkit/linuxkit/src/cmd/linuxkit/version"
+	"github.com/moby/tool/src/moby"
 
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 )
 
+func init() {
+	// Register LinuxKit images to build outputs with the vendored moby tool.
+	// This allows us to overwrite the hashes locally without having
+	// to re-vendor the 'github.com/moby/tool' when we update 'mkimage-*'
+	imgs := map[string]string{
+		"iso-bios":    "linuxkit/mkimage-iso-bios:9a51dc64a461f1cc50ba05f30a38f73f5227ac03",
+		"iso-efi":     "linuxkit/mkimage-iso-efi:343cf1a8ac0aba7d8a1f13b7f45fa0b57ab897dc",
+		"raw-bios":    "linuxkit/mkimage-raw-bios:d90713b2dd610cf9a0f5f9d9095f8bf86f40d5c6",
+		"raw-efi":     "linuxkit/mkimage-raw-efi:8938ffb6014543e557b624a40cce1714f30ce4b6",
+		"squashfs":    "linuxkit/mkimage-squashfs:b44d00b0a336fd32c122ff32bd2b39c36a965135",
+		"gcp":         "linuxkit/mkimage-gcp:e6cdcf859ab06134c0c37a64ed5f886ec8dae1a1",
+		"qcow2-efi":   "linuxkit/mkimage-qcow2-efi:787b54906e14a56b9f1da35dcc8e46bd58435285",
+		"vhd":         "linuxkit/mkimage-vhd:3820219e5c350fe8ab2ec6a217272ae82f4b9242",
+		"dynamic-vhd": "linuxkit/mkimage-dynamic-vhd:743ac9959fe6d3912ebd78b4fd490b117c53f1a6",
+		"vmdk":        "linuxkit/mkimage-vmdk:cee81a3ed9c44ae446ef7ebff8c42c1e77b3e1b5",
+		"rpi3":        "linuxkit/mkimage-rpi3:0f23c4f37cdca99281ca33ac6188e1942fa7a2b8",
+	}
+	if err := moby.UpdateOutputImages(imgs); err != nil {
+		log.Fatalf("Failed to register mkimage-*. %v", err)
+	}
+}
+
 // GlobalConfig is the global tool configuration
 type GlobalConfig struct {
 	Pkg PkgConfig `yaml:"pkg"`

src/cmd/linuxkit/run_hyperkit.go

@@ -66,6 +66,7 @@ func runHyperKit(args []string) {
 	// Boot type; we try to determine automatically
 	uefiBoot := flags.Bool("uefi", false, "Use UEFI boot")
 	isoBoot := flags.Bool("iso", false, "Boot image is an ISO")
+	squashFSBoot := flags.Bool("squashfs", false, "Boot image is a kernel+squashfs+cmdline")
 	kernelBoot := flags.Bool("kernel", false, "Boot image is kernel+initrd+cmdline 'path'-kernel/-initrd/-cmdline")
 
 	// Hyperkit settings
@@ -87,55 +88,54 @@ func runHyperKit(args []string) {
 	path := remArgs[0]
 	prefix := path
 
-	info, err := os.Stat(path)
+	_, err := os.Stat(path + "-kernel")
-	stat := err == nil
-
-	// ignore a directory
-	if stat && info.Mode().IsDir() {
-		stat = false
-	}
-
-	_, err = os.Stat(path + "-kernel")
 	statKernel := err == nil
 
 	var isoPaths []string
 
-	// try to autodetect boot type if not specified
-	// if the path does not exist, and the kernel does, must be trying to do a kernel boot
-	// if the path does exist and ends in ISO, must be trying ISO boot
-	if !stat && statKernel && !*isoBoot {
-		*kernelBoot = true
-	} else if stat && strings.HasSuffix(path, ".iso") && !*kernelBoot {
-		*isoBoot = true
-	}
-
 	switch {
-	case *kernelBoot && *isoBoot:
+	case *squashFSBoot:
-		log.Fatalf("Cannot specify both kernel and ISO boot together")
+		if *kernelBoot || *isoBoot {
-	case *kernelBoot:
+			log.Fatalf("Please specify only one boot method")
+		}
 		if !statKernel {
-			log.Fatalf("Cannot find kernel file (%s): %v", path+"-kernel", err)
+			log.Fatalf("Booting a SquashFS root filesystem requires a kernel at %s", path+"-kernel")
+		}
+		_, err = os.Stat(path + "-squashfs.img")
+		statSquashFS := err == nil
+		if !statSquashFS {
+			log.Fatalf("Cannot find SquashFS image (%s): %v", path+"-squashfs.img", err)
+		}
+	case *isoBoot:
+		if *kernelBoot {
+			log.Fatalf("Please specify only one boot method")
+		}
+		if !*uefiBoot {
+			log.Fatalf("Hyperkit requires --uefi to be set to boot an ISO")
+		}
+		// We used to auto-detect ISO boot. For backwards compat, append .iso if not present
+		isoPath := path
+		if !strings.HasSuffix(isoPath, ".iso") {
+			isoPath += ".iso"
+		}
+		_, err = os.Stat(isoPath)
+		statISO := err == nil
+		if !statISO {
+			log.Fatalf("Cannot find ISO image (%s): %v", isoPath, err)
+		}
+		prefix = strings.TrimSuffix(path, ".iso")
+		isoPaths = append(isoPaths, isoPath)
+	default:
+		// Default to kernel+initrd
+		if !statKernel {
+			log.Fatalf("Cannot find kernel file: %s", path+"-kernel")
 		}
 		_, err = os.Stat(path + "-initrd.img")
 		statInitrd := err == nil
 		if !statInitrd {
 			log.Fatalf("Cannot find initrd file (%s): %v", path+"-initrd.img", err)
 		}
-	case *isoBoot:
+		*kernelBoot = true
-		if !stat {
-			log.Fatalf("Cannot find ISO to boot")
-		}
-		prefix = strings.TrimSuffix(path, ".iso")
-		// hyperkit only supports UEFI ISO boot at present
-		if !*uefiBoot {
-			log.Fatalf("Hyperkit requires --uefi to be set to boot an ISO")
-		}
-		isoPaths = append(isoPaths, path)
-	default:
-		if !stat {
-			log.Fatalf("Cannot find file %s to boot", path)
-		}
-		log.Fatalf("Unrecognised boot type, please specify on command line")
 	}
 
 	if *uefiBoot {
@@ -186,12 +186,13 @@ func runHyperKit(args []string) {
 	vmUUID := uuid.New().String()
 
 	// Run
-	var cmdline []byte
+	var cmdline string
-	if *kernelBoot {
+	if *kernelBoot || *squashFSBoot {
-		cmdline, err = ioutil.ReadFile(prefix + "-cmdline")
+		cmdlineBytes, err := ioutil.ReadFile(prefix + "-cmdline")
 		if err != nil {
 			log.Fatalf("Cannot open cmdline file: %v", err)
 		}
+		cmdline = string(cmdlineBytes)
 	}
 
 	// Create new HyperKit instance (w/o networking for now)
@@ -204,6 +205,28 @@ func runHyperKit(args []string) {
 		h.Console = hyperkit.ConsoleFile
 	}
 
+	h.UUID = vmUUID
+	h.ISOImages = isoPaths
+	h.VSock = true
+	h.CPUs = *cpus
+	h.Memory = *mem
+
+	switch {
+	case *kernelBoot:
+		h.Kernel = prefix + "-kernel"
+		h.Initrd = prefix + "-initrd.img"
+	case *squashFSBoot:
+		h.Kernel = prefix + "-kernel"
+		// Make sure the SquashFS image is the first disk, raw, and virtio
+		var rootDisk hyperkit.RawDisk
+		rootDisk.Path = prefix + "-squashfs.img"
+		rootDisk.Trim = false // This happens to select 'virtio-blk'
+		h.Disks = append(h.Disks, &rootDisk)
+		cmdline = cmdline + " root=/dev/vda"
+	default:
+		h.Bootrom = *fw
+	}
+
 	for i, d := range disks {
 		id := ""
 		if i != 0 {
@@ -276,18 +299,6 @@ func runHyperKit(args []string) {
 		log.Fatalf("Invalid networking mode: %s", netMode[0])
 	}
 
-	if *kernelBoot {
-		h.Kernel = prefix + "-kernel"
-		h.Initrd = prefix + "-initrd.img"
-	} else {
-		h.Bootrom = *fw
-	}
-	h.UUID = vmUUID
-	h.ISOImages = isoPaths
-	h.VSock = true
-	h.CPUs = *cpus
-	h.Memory = *mem
-
 	h.VPNKitUUID = *vpnkitUUID
 	if *ipStr != "" {
 		if ip := net.ParseIP(*ipStr); len(ip) > 0 && ip.To4() != nil {
@@ -315,7 +326,7 @@ func runHyperKit(args []string) {
 		}
 	}
 
-	err = h.Run(string(cmdline))
+	err = h.Run(cmdline)
 	if err != nil {
 		log.Fatalf("Cannot run hyperkit: %v", err)
 	}

src/cmd/linuxkit/run_qemu.go

@@ -29,6 +29,7 @@ type QemuConfig struct {
 	Path           string
 	ISOBoot        bool
 	UEFI           bool
+	SquashFS       bool
 	Kernel         bool
 	GUI            bool
 	Disks          Disks
@@ -139,6 +140,7 @@ func runQemu(args []string) {
 	// Boot type; we try to determine automatically
 	uefiBoot := flags.Bool("uefi", false, "Use UEFI boot")
 	isoBoot := flags.Bool("iso", false, "Boot image is an ISO")
+	squashFSBoot := flags.Bool("squashfs", false, "Boot image is a kernel+squashfs+cmdline")
 	kernelBoot := flags.Bool("kernel", false, "Boot image is kernel+initrd+cmdline 'path'-kernel/-initrd/-cmdline")
 
 	// State flags
@@ -199,12 +201,18 @@ func runQemu(args []string) {
 	_, err := os.Stat(path)
 	stat := err == nil
 
-	// if the path does not exist, must be trying to do a kernel boot
+	// if the path does not exist, must be trying to do a kernel+initrd or kernel+squashfs boot
 	if !stat {
 		_, err = os.Stat(path + "-kernel")
 		statKernel := err == nil
 		if statKernel {
-			*kernelBoot = true
+			_, err = os.Stat(path + "-squashfs.img")
+			statSquashFS := err == nil
+			if statSquashFS {
+				*squashFSBoot = true
+			} else {
+				*kernelBoot = true
+			}
 		}
 		// we will error out later if neither found
 	} else {
@@ -252,13 +260,19 @@ func runQemu(args []string) {
 		disks[i] = d
 	}
 
-	// user not trying to boot off ISO or kernel, so assume booting from a disk image
+	// user not trying to boot off ISO or kernel+initrd, so assume booting from a disk image or kernel+squashfs
 	if !*kernelBoot && !*isoBoot {
-		if _, err := os.Stat(path); err != nil {
+		var diskPath string
-			log.Fatalf("Boot disk image %s does not exist", path)
+		if *squashFSBoot {
+			diskPath = path + "-squashfs.img"
+		} else {
+			if _, err := os.Stat(path); err != nil {
+				log.Fatalf("Boot disk image %s does not exist", path)
+			}
+			diskPath = path
 		}
 		// currently no way to set format, but autodetect probably works
-		d := Disks{DiskConfig{Path: path}}
+		d := Disks{DiskConfig{Path: diskPath}}
 		disks = append(d, disks...)
 	}
 
@@ -301,6 +315,7 @@ func runQemu(args []string) {
 		Path:           path,
 		ISOBoot:        *isoBoot,
 		UEFI:           *uefiBoot,
+		SquashFS:       *squashFSBoot,
 		Kernel:         *kernelBoot,
 		GUI:            *enableGUI,
 		Disks:          disks,
@@ -603,16 +618,28 @@ func buildQemuCmdline(config QemuConfig) (QemuConfig, []string) {
 	}
 
 	// build kernel boot config from kernel/initrd/cmdline
-	if config.Kernel {
+	switch {
+	case config.Kernel:
 		qemuKernelPath := config.Path + "-kernel"
 		qemuInitrdPath := config.Path + "-initrd.img"
 		qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath)
 		qemuArgs = append(qemuArgs, "-initrd", qemuInitrdPath)
-		cmdlineString, err := ioutil.ReadFile(config.Path + "-cmdline")
+		cmdlineBytes, err := ioutil.ReadFile(config.Path + "-cmdline")
 		if err != nil {
 			log.Errorf("Cannot open cmdline file: %v", err)
 		} else {
-			qemuArgs = append(qemuArgs, "-append", string(cmdlineString))
+			qemuArgs = append(qemuArgs, "-append", string(cmdlineBytes))
+		}
+	case config.SquashFS:
+		qemuKernelPath := config.Path + "-kernel"
+		qemuArgs = append(qemuArgs, "-kernel", qemuKernelPath)
+		cmdlineBytes, err := ioutil.ReadFile(config.Path + "-cmdline")
+		if err != nil {
+			log.Errorf("Cannot open cmdline file: %v", err)
+		} else {
+			cmdline := string(cmdlineBytes)
+			cmdline += " root=/dev/sda"
+			qemuArgs = append(qemuArgs, "-append", cmdline)
 		}
 	}
 

src/cmd/linuxkit/vendor.conf

@@ -23,10 +23,10 @@ github.com/gorilla/mux 4c1c3952b7d9d0a061a3fa7b36fd373ba0398ebc
 github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
 github.com/mitchellh/go-ps 4fdf99ab29366514c69ccccddab5dc58b8d84062
 github.com/moby/datakit 97b3d230535397a813323902c23751e176481a86
-github.com/moby/hyperkit a285521725f44f3d10ca1042c2c07d3a6e24bed8
+github.com/moby/hyperkit d65b09c1c28a2bfb6a976c86ecd885d2ee4c71d3
 # When updating also:
 # curl -fsSL -o src/cmd/linuxkit/build.go https://raw.githubusercontent.com/moby/tool/«hash»/cmd/moby/build.go
-github.com/moby/tool 486e313fe3069da88948500ceb60d0abfffad464
+github.com/moby/tool 3dbad3b7daffd631d036493a1e883608206d2e03
 github.com/moby/vpnkit 0e4293bb1058598c4b0a406ed171f52573ef414c
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448
 github.com/opencontainers/image-spec v1.0.0

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/README.md

@@ -1,10 +1,10 @@
 ## [HyperKit](http://github.com/moby/hyperkit)
 
-![Build Status OSX](https://circleci.com/gh/moby/hyperkit.svg?style=shield&circle-token=cf8379b302eab2bbf33821cafe164dbefb71982d)
+![Build Status macOS](https://circleci.com/gh/moby/hyperkit.svg?style=shield&circle-token=cf8379b302eab2bbf33821cafe164dbefb71982d)
 
 *HyperKit* is a toolkit for embedding hypervisor capabilities in your application. It includes a complete hypervisor, based on [xhyve](https://github.com/mist64/xhyve)/[bhyve](http://bhyve.org), which is optimized for lightweight virtual machines and container deployment.  It is designed to be interfaced with higher-level components such as the [VPNKit](https://github.com/moby/vpnkit) and [DataKit](https://github.com/moby/datakit).
 
-HyperKit currently only supports Mac OS X using the [Hypervisor.framework](https://developer.apple.com/library/mac/documentation/DriversKernelHardware/Reference/Hypervisor/index.html). It is a core component of Docker For Mac.
+HyperKit currently only supports macOS using the [Hypervisor.framework](https://developer.apple.com/library/mac/documentation/DriversKernelHardware/Reference/Hypervisor/index.html). It is a core component of Docker For Mac.
 
 
 ## Requirements

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/go/disk.go

@@ -65,7 +65,7 @@ func GetDiskFormat(path string) DiskFormat {
 	case ".raw", ".img":
 		return DiskFormatRaw
 	default:
-		log.Debugf("Unknown disk extension %q, will use raw format", path)
+		log.Debugf("hyperkit: Unknown disk extension %q, will use raw format", path)
 		return DiskFormatRaw
 	}
 }
@@ -96,7 +96,7 @@ func NewDisk(spec string, size int) (Disk, error) {
 func exists(d Disk) bool {
 	_, err := os.Stat(d.GetPath())
 	if err != nil && !os.IsNotExist(err) {
-		log.Debugf("cannot stat %q: %v", d, err)
+		log.Debugf("hyperkit: cannot stat %q: %v", d, err)
 	}
 	return err == nil
 }
@@ -114,7 +114,7 @@ func ensure(d Disk) error {
 		return d.resize()
 	}
 	if d.GetSize() < current {
-		log.Errorf("Cannot safely shrink %q from %dMiB to %dMiB", d, current, d.GetSize())
+		log.Errorf("hyperkit: Cannot safely shrink %q from %dMiB to %dMiB", d, current, d.GetSize())
 	}
 	return nil
 }
@@ -199,7 +199,7 @@ func (d *RawDisk) Ensure() error {
 
 // Create a disk.
 func (d *RawDisk) create() error {
-	log.Infof("Create %q", d)
+	log.Infof("hyperkit: Create %q", d)
 	f, err := os.Create(d.Path)
 	if err != nil {
 		return err
@@ -225,7 +225,7 @@ func (d *RawDisk) resize() error {
 	if err != nil {
 		return fmt.Errorf("Cannot resize %q: %v", d, err)
 	}
-	log.Infof("Resize %q from %vMiB to %vMiB", d, s, d.GetSize())
+	log.Infof("hyperkit: Resize %q from %vMiB to %vMiB", d, s, d.GetSize())
 	// APFS exhibits a weird behavior wrt sparse files: we cannot
 	// create (or grow) them "too fast": there's a limit,
 	// apparently related to the available disk space.  However,
@@ -241,7 +241,7 @@ func (d *RawDisk) resize() error {
 			return fmt.Errorf("Cannot resize %q to %vMiB: %v", d, s, err)
 		}
 	}
-	log.Infof("Resized %q to %vMiB", d, d.GetSize())
+	log.Infof("hyperkit: Resized %q to %vMiB", d, d.GetSize())
 	return nil
 }
 
@@ -314,7 +314,7 @@ func (d *QcowDisk) QcowTool(verb string, args ...string) *exec.Cmd {
 func run(cmd *exec.Cmd) (string, error) {
 	buf, err := cmd.CombinedOutput()
 	out := string(buf)
-	log.Debugf("ran %v: out=%q, err=%v", cmd.Args, out, err)
+	log.Debugf("hyperkit: ran %v: out=%q, err=%v", cmd.Args, out, err)
 	return out, err
 }
 
@@ -326,18 +326,18 @@ func (d *QcowDisk) Exists() bool {
 // Ensure creates the disk image if needed, and resizes it if needed.
 func (d *QcowDisk) Ensure() error {
 	if d.Trim {
-		log.Infof("%v: TRIM is enabled; recycling thread will keep %v sectors free and will compact after %v more sectors are free",
+		log.Infof("hyperkit: %v: TRIM is enabled; recycling thread will keep %v sectors free and will compact after %v more sectors are free",
 			d, d.KeepErased, d.CompactAfter)
 	}
 	if d.RuntimeAsserts {
-		log.Warnf("%v: Expensive runtime checks are enabled", d)
+		log.Warnf("hyperkit: %v: Expensive runtime checks are enabled", d)
 	}
 	return ensure(d)
 }
 
 // Create a disk with the given size in MiB
 func (d *QcowDisk) create() error {
-	log.Infof("Create %q", d)
+	log.Infof("hyperkit: Create %q", d)
 	_, err := run(d.QcowTool("create", "--size", fmt.Sprintf("%dMiB", d.Size)))
 	return err
 }
@@ -368,14 +368,14 @@ func (d *QcowDisk) sizeString() string {
 
 // Resize the virtual size of the disk
 func (d *QcowDisk) resize() error {
-	log.Infof("Resize %q from %v to %dMiB", d, d.sizeString(), d.GetSize())
+	log.Infof("hyperkit: Resize %q from %v to %dMiB", d, d.sizeString(), d.GetSize())
 	_, err := run(d.QcowTool("resize", "--size", fmt.Sprintf("%dMiB", d.Size)))
 	return err
 }
 
 // compact the disk to shrink the physical size.
 func (d *QcowDisk) compact() error {
-	log.Infof("Compact: %q... (%v)", d, d.sizeString())
+	log.Infof("hyperkit: Compact: %q... (%v)", d, d.sizeString())
 	cmd := d.QcowTool("compact")
 	if _, err := run(cmd); err != nil {
 		if err.(*exec.ExitError) != nil {
@@ -383,7 +383,7 @@ func (d *QcowDisk) compact() error {
 		}
 		return err
 	}
-	log.Infof("Compact: %q: done (%v)", d, d.sizeString())
+	log.Infof("hyperkit: Compact: %q: done (%v)", d, d.sizeString())
 	return nil
 }
 
@@ -402,14 +402,14 @@ func (d *QcowDisk) check() error {
 // Stop cleans up this disk when we are quitting.
 func (d *QcowDisk) Stop() error {
 	if !d.Trim && d.CompactAfter == 0 {
-		log.Infof("TRIM is enabled but auto-compaction disabled: compacting %q now", d)
+		log.Infof("hyperkit: TRIM is enabled but auto-compaction disabled: compacting %q now", d)
 		if err := d.compact(); err != nil {
 			return fmt.Errorf("Failed to compact %q: %v", d, err)
 		}
 		if err := d.check(); err != nil {
 			return fmt.Errorf("Post-compact disk integrity check of %q failed: %v", d, err)
 		}
-		log.Infof("Post-compact disk integrity check of %q successful", d)
+		log.Infof("hyperkit: Post-compact disk integrity check of %q successful", d)
 	}
 	return nil
 }

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/go/hyperkit.go

@@ -44,6 +44,8 @@ const (
 	ConsoleStdio = iota
 	// ConsoleFile configures console to a tty and output to a file
 	ConsoleFile
+	// ConsoleLog configures console to a tty and sends its contents to the logs
+	ConsoleLog
 
 	defaultVPNKitSock = "Library/Containers/com.docker.docker/Data/s50"
 
@@ -117,8 +119,7 @@ type HyperKit struct {
 	// Memory is the amount of megabytes of memory for the VM.
 	Memory int `json:"memory"`
 
-	// Console defines where the console of the VM should be
+	// Console defines where the console of the VM should be connected to.
-	// connected to. ConsoleStdio and ConsoleFile are supported.
 	Console int `json:"console"`
 
 	// Below here are internal members, but they are exported so
@@ -199,11 +200,15 @@ func (h *HyperKit) check() error {
 	log.Debugf("hyperkit: check %#v", h)
 	var err error
 	// Sanity checks on configuration
-	if h.Console == ConsoleFile && h.StateDir == "" {
+	switch h.Console {
-		return fmt.Errorf("If ConsoleFile is set, StateDir must be specified")
+	case ConsoleFile, ConsoleLog:
-	}
+		if h.StateDir == "" {
-	if h.Console == ConsoleStdio && !isTerminal(os.Stdout) && h.StateDir == "" {
+			return fmt.Errorf("If ConsoleFile or ConsoleLog is set, StateDir must be specified")
-		return fmt.Errorf("If ConsoleStdio is set but stdio is not a terminal, StateDir must be specified")
+		}
+	case ConsoleStdio:
+		if !isTerminal(os.Stdout) && h.StateDir == "" {
+			return fmt.Errorf("If ConsoleStdio is set but stdio is not a terminal, StateDir must be specified")
+		}
 	}
 	for _, image := range h.ISOImages {
 		if _, err = os.Stat(image); os.IsNotExist(err) {
@@ -226,8 +231,10 @@ func (h *HyperKit) check() error {
 		if _, err = os.Stat(h.Kernel); os.IsNotExist(err) {
 			return fmt.Errorf("Kernel %s does not exist", h.Kernel)
 		}
-		if _, err = os.Stat(h.Initrd); os.IsNotExist(err) {
+		if h.Initrd != "" {
-			return fmt.Errorf("initrd %s does not exist", h.Initrd)
+			if _, err = os.Stat(h.Initrd); os.IsNotExist(err) {
+				return fmt.Errorf("initrd %s does not exist", h.Initrd)
+			}
 		}
 	} else {
 		if _, err = os.Stat(h.Bootrom); os.IsNotExist(err) {
@@ -420,10 +427,20 @@ func (h *HyperKit) buildArgs(cmdline string) {
 		nextSlot++
 	}
 
-	if h.Console == ConsoleStdio && isTerminal(os.Stdout) {
+	// -l: LPC device configuration.
-		a = append(a, "-l", "com1,stdio")
+	{
-	} else if h.StateDir != "" {
+		cfg := "com1"
-		a = append(a, "-l", fmt.Sprintf("com1,autopty=%s/tty,log=%s/console-ring", h.StateDir, h.StateDir))
+		if h.Console == ConsoleStdio && isTerminal(os.Stdout) {
+			cfg += fmt.Sprintf(",stdio")
+		} else {
+			cfg += fmt.Sprintf(",autopty=%s/tty", h.StateDir)
+		}
+		if h.Console == ConsoleLog {
+			cfg += fmt.Sprintf(",asl")
+		} else {
+			cfg += fmt.Sprintf(",log=%s/console-ring", h.StateDir)
+		}
+		a = append(a, "-l", cfg)
 	}
 
 	if h.Bootrom == "" {
@@ -440,6 +457,22 @@ func (h *HyperKit) buildArgs(cmdline string) {
 	log.Debugf("hyperkit: CmdLine: %#v", h.CmdLine)
 }
 
+// openTTY opens the tty files for reading, and returns it.
+func (h *HyperKit) openTTY() *os.File {
+	path := fmt.Sprintf("%s/tty", h.StateDir)
+	for {
+		if res, err := os.OpenFile(path, os.O_RDONLY, 0); err != nil {
+			log.Infof("hyperkit: openTTY: %v, retrying", err)
+			time.Sleep(10 * time.Millisecond)
+		} else {
+			log.Infof("hyperkit: openTTY: got %v", path)
+			saneTerminal(res)
+			setRaw(res)
+			return res
+		}
+	}
+}
+
 // execute forges the command to run hyperkit, runs and returns it.
 // It also plumbs stdin/stdout/stderr.
 func (h *HyperKit) execute() (*exec.Cmd, error) {
@@ -469,26 +502,13 @@ func (h *HyperKit) execute() (*exec.Cmd, error) {
 			cmd.Stderr = os.Stderr
 		} else {
 			go func() {
-				ttyPath := fmt.Sprintf("%s/tty", h.StateDir)
+				tty := h.openTTY()
-				var tty *os.File
+				defer tty.Close()
-				for {
-					var err error
-					tty, err = os.OpenFile(ttyPath, os.O_RDONLY, 0)
-					if err == nil {
-						break
-					}
-					time.Sleep(10 * time.Millisecond)
-				}
-				saneTerminal(tty)
-				setRaw(tty)
 				io.Copy(os.Stdout, tty)
-				tty.Close()
 			}()
 		}
 	} else if log != nil {
 		log.Debugf("hyperkit: Redirecting stdout/stderr to logger")
-		stdoutChan := make(chan string)
-		stderrChan := make(chan string)
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
 			return nil, err
@@ -497,22 +517,8 @@ func (h *HyperKit) execute() (*exec.Cmd, error) {
 		if err != nil {
 			return nil, err
 		}
-		stream(stdout, stdoutChan)
+		go logStream(stdout, "stdout")
-		stream(stderr, stderrChan)
+		go logStream(stderr, "stderr")
-
-		done := make(chan struct{})
-		go func() {
-			for {
-				select {
-				case stderrl := <-stderrChan:
-					log.Infof("%s", stderrl)
-				case stdoutl := <-stdoutChan:
-					log.Infof("%s", stdoutl)
-				case <-done:
-					return
-				}
-			}
-		}()
 	}
 
 	log.Debugf("hyperkit: Starting %#v", cmd)
@@ -546,18 +552,18 @@ func (h *HyperKit) writeState() error {
 	return ioutil.WriteFile(filepath.Join(h.StateDir, jsonFile), []byte(s), 0644)
 }
 
-func stream(r io.ReadCloser, dest chan<- string) {
+// logStream redirects a file to the logs.
-	go func() {
+func logStream(r io.ReadCloser, name string) {
-		defer r.Close()
+	defer r.Close()
-		reader := bufio.NewReader(r)
+	reader := bufio.NewReader(r)
-		for {
+	for {
-			line, err := reader.ReadString('\n')
+		line, err := reader.ReadString('\n')
-			if err != nil {
+		if err != nil {
-				return
+			log.Warnf("hyperkit: failed to read %v: %v", name, err)
-			}
+			break
-			dest <- line
 		}
-	}()
+		log.Infof("hyperkit: %v: %v", name, line)
+	}
 }
 
 // checkHyperKit tries to find and/or validate the path of hyperkit

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/go/pty_util_fallback.go

@@ -7,17 +7,17 @@ import (
 )
 
 func saneTerminal(f *os.File) error {
-	log.Fatalf("Function not supported on your OS")
+	log.Fatalf("hyperkit: Function not supported on your OS")
 	return nil
 }
 
 func setRaw(f *os.File) error {
-	log.Fatalf("Function not supported on your OS")
+	log.Fatalf("hyperkit: Function not supported on your OS")
 	return nil
 }
 
 // isTerminal checks if the provided file is a terminal
 func isTerminal(f *os.File) bool {
-	log.Fatalf("Function not supported on your OS")
+	log.Fatalf("hyperkit: Function not supported on your OS")
 	return false
 }

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/include/xhyve/log.h

@@ -0,0 +1,7 @@
+#pragma once
+
+/* Initialize ASL logger and local buffer. */
+void log_init(void);
+
+/* Send one character to the logger: wait for full lines before actually sending. */
+void log_put(uint8_t _c);

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/lib/block_if.c

@@ -257,27 +257,40 @@ block_delete(struct blockif_ctxt *bc, off_t offset, off_t len)
 
 		size_t aligned_offset = ALIGNUP(fp_offset, bc->bc_delete_alignment);
 		if (aligned_offset != fp_offset) {
-			size_t len_to_zero = aligned_offset - fp_offset;
+			size_t len_to_zero = MIN(fp_length, aligned_offset - fp_offset);
-			assert(len_to_zero <= bc->bc_delete_alignment);
+			assert(len_to_zero < bc->bc_delete_alignment);
-			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf, len_to_zero, (off_t) fp_offset);
+			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf,
+				len_to_zero, (off_t)fp_offset);
 			if (written == -1) goto out;
 			fp_offset += len_to_zero;
 			fp_length -= len_to_zero;
 		}
 		size_t aligned_length = ALIGNDOWN(fp_length, bc->bc_delete_alignment);
-		if (aligned_length != fp_length) {
+		if (aligned_length >= bc->bc_delete_alignment) {
-			size_t len_to_zero = fp_length - aligned_length;
+			assert(fp_offset % bc->bc_delete_alignment == 0);
-			assert(len_to_zero <= bc->bc_delete_alignment);
+			struct fpunchhole arg = {
-			fp_length -= len_to_zero;
+				.fp_flags = 0,
-			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf, len_to_zero, (off_t) (fp_offset + fp_length));
+				.reserved = 0,
+				.fp_offset = (off_t)fp_offset,
+				.fp_length = (off_t)aligned_length
+			};
+			int punched = fcntl(bc->bc_fd, F_PUNCHHOLE, &arg);
+			if (punched == -1) goto out;
+			fp_offset += aligned_length;
+			fp_length -= aligned_length;
+		}
+		if (fp_length > 0) {
+			assert(fp_length < bc->bc_delete_alignment);
+			assert(fp_offset % bc->bc_delete_alignment == 0);
+			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf,
+				fp_length, (off_t)fp_offset);
 			if (written == -1) goto out;
 		}
-		struct fpunchhole arg = { .fp_flags = 0, .reserved = 0, .fp_offset = (off_t) fp_offset, .fp_length = (off_t) fp_length };
+		ret = 0;
-		ret = fcntl(bc->bc_fd, F_PUNCHHOLE, &arg);
-		goto out;
 	}
-#endif
+#else
 	errno = EOPNOTSUPP;
+#endif
 out:
 	HYPERKIT_BLOCK_DELETE_DONE(offset, ret);
 	return ret;

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/lib/log.c

@@ -0,0 +1,50 @@
+#include <asl.h>
+#include <pwd.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <time.h>
+
+#include <SystemConfiguration/SystemConfiguration.h>
+
+#include <xhyve/log.h>
+
+static aslclient log_client = NULL;
+static aslmsg log_msg = NULL;
+
+static unsigned char buf[4096];
+/* Index of the _next_ character to insert in the buffer. */
+static size_t buf_idx = 0;
+
+/* asl is deprecated in favor of os_log starting with macOS 10.12.  */
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+/* Initialize ASL logger and local buffer. */
+void log_init(void)
+{
+	log_client = asl_open(NULL, NULL, 0);
+	log_msg = asl_new(ASL_TYPE_MSG);
+}
+
+
+/* Send the content of the buffer to the logger. */
+static void log_flush(void)
+{
+	buf[buf_idx] = 0;
+	asl_log(log_client, log_msg, ASL_LEVEL_NOTICE, "%s", buf);
+	buf_idx = 0;
+}
+
+
+/* Send one character to the logger: wait for full lines before actually sending. */
+void log_put(uint8_t c)
+{
+	if ((c == '\n') || (c == 0)) {
+		log_flush();
+	} else {
+		if (buf_idx + 2 >= sizeof(buf)) {
+			log_flush();
+		}
+		buf[buf_idx] = c;
+		++buf_idx;
+	}
+}

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/lib/uart_emul.c

@@ -41,13 +41,14 @@
 #include <assert.h>
 #include <errno.h>
 #include <sys/mman.h>
+#include <xhyve/log.h>
 #include <xhyve/support/ns16550.h>
 #include <xhyve/mevent.h>
 #include <xhyve/uart_emul.h>
 
-#define	COM1_BASE      	0x3F8
+#define	COM1_BASE	0x3F8
 #define COM1_IRQ	4
-#define	COM2_BASE      	0x2F8
+#define	COM2_BASE	0x2F8
 #define COM2_IRQ	3
 
 #define	DEFAULT_RCLK	1843200
@@ -91,7 +92,7 @@ struct fifo {
 struct ttyfd {
 	bool	opened;
 	int	fd;		/* tty device file descriptor */
-	int 	sfd;
+	int	sfd;
 	char *name; /* slave pty name when using autopty*/
 	struct termios tio_orig, tio_new;    /* I/O Terminals */
 };
@@ -121,6 +122,7 @@ struct uart_softc {
 
 	struct ttyfd tty;
 	struct log log;
+	bool	asl;		/* Output to Apple logger. */
 	bool	thre_int_pending;	/* THRE interrupt pending */
 
 	void	*arg;
@@ -427,7 +429,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 		}
 	}
 
-        switch (offset) {
+	switch (offset) {
 	case REG_DATA:
 		if (sc->mcr & MCR_LOOPBACK) {
 			if (rxfifo_putchar(sc, value) != 0)
@@ -436,6 +438,8 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 			ttywrite(&sc->tty, value);
 			if (sc->log.ring)
 				ringwrite(&sc->log, value);
+			if (sc->asl)
+				log_put(value);
 		} /* else drop on floor */
 		sc->thre_int_pending = true;
 		break;
@@ -681,15 +685,15 @@ uart_tty_backend(struct uart_softc *sc, const char *backend)
 static char *
 copy_up_to_comma(const char *from)
 {
-        char *comma = strchr(from, ',');
+	char *comma = strchr(from, ',');
-        char *tmp = NULL;
+	char *tmp = NULL;
-        if (comma == NULL) {
+	if (comma == NULL) {
-                tmp = strdup(from); /* rest of string */
+		tmp = strdup(from); /* rest of string */
-        } else {
+	} else {
-                ptrdiff_t length = comma - from;
+		ptrdiff_t length = comma - from;
-                tmp = strndup(from, (size_t)length);
+		tmp = strndup(from, (size_t)length);
-        }
+	}
-        return tmp;
+	return tmp;
 }
 
 int
@@ -773,6 +777,10 @@ uart_set_backend(struct uart_softc *sc, const char *backend, const char *devname
 			if (uart_mapring(sc, logname) == -1) {
 				goto err;
 			}
+		} else if (strcmp("asl", backend) == 0) {
+			sc->asl = true;
+			log_init();
+			retval = 0;
 		} else if (uart_tty_backend(sc, backend) == 0) {
 			retval = 0;
 		} else {

src/cmd/linuxkit/vendor/github.com/moby/tool/src/initrd/initrd.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"io/ioutil"
 	"path/filepath"
+	"strings"
 
 	"github.com/moby/tool/src/pad4"
 	"github.com/surma/gocpio"
@@ -121,26 +122,26 @@ func CopySplitTar(w *Writer, r *tar.Reader) (kernel []byte, cmdline string, ucod
 		if err != nil {
 			return
 		}
-		switch thdr.Name {
+		switch {
-		case "boot/kernel":
+		case thdr.Name == "boot/kernel":
 			kernel, err = ioutil.ReadAll(r)
 			if err != nil {
 				return
 			}
-		case "boot/cmdline":
+		case thdr.Name == "boot/cmdline":
 			var buf []byte
 			buf, err = ioutil.ReadAll(r)
 			if err != nil {
 				return
 			}
 			cmdline = string(buf)
-		case "boot/ucode.cpio":
+		case thdr.Name == "boot/ucode.cpio":
 			ucode, err = ioutil.ReadAll(r)
 			if err != nil {
 				return
 			}
-		case "boot":
+		case strings.HasPrefix(thdr.Name, "boot/"):
-			// skip this entry
+			// skip the rest of ./boot
 		default:
 			_, err = copyTarEntry(w, thdr, r)
 			if err != nil {

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/docker.go

@@ -22,7 +22,7 @@ import (
 )
 
 func dockerRun(input io.Reader, output io.Writer, trust bool, img string, args ...string) error {
-	log.Debugf("docker run (input): %s", strings.Join(args, " "))
+	log.Debugf("docker run %s (trust=%t) (input): %s", img, trust, strings.Join(args, " "))
 	docker, err := exec.LookPath("docker")
 	if err != nil {
 		return errors.New("Docker does not seem to be installed")
@@ -38,7 +38,7 @@ func dockerRun(input io.Reader, output io.Writer, trust bool, img string, args .
 	pull.Env = env
 	if err := pull.Run(); err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {
-			return fmt.Errorf("docker pull failed: %v output:\n%s", err, exitError.Stderr)
+			return fmt.Errorf("docker pull %s failed: %v output:\n%s", img, err, exitError.Stderr)
 		}
 		return err
 	}
@@ -51,12 +51,12 @@ func dockerRun(input io.Reader, output io.Writer, trust bool, img string, args .
 
 	if err := cmd.Run(); err != nil {
 		if exitError, ok := err.(*exec.ExitError); ok {
-			return fmt.Errorf("docker run failed: %v output:\n%s", err, exitError.Stderr)
+			return fmt.Errorf("docker run %s failed: %v output:\n%s", img, err, exitError.Stderr)
 		}
 		return err
 	}
 
-	log.Debugf("docker run (input): %s...Done", strings.Join(args, " "))
+	log.Debugf("docker run %s (input): %s...Done", img, strings.Join(args, " "))
 	return nil
 }
 

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/linuxkit.go

@@ -17,11 +17,11 @@ kernel:
   image: linuxkit/kernel:4.9.39
   cmdline: "console=ttyS0"
 init:
-  - linuxkit/init:v0.3
+  - linuxkit/init:00ab58c9681a0bf42b2e35134c1ccf1591ebb64d
-  - linuxkit/runc:v0.3
+  - linuxkit/runc:f5960b83a8766ae083efc744fa63dbf877450e4f
 onboot:
   - name: mkimage
-    image: linuxkit/mkimage:v0.3
+    image: linuxkit/mkimage:50bde8b00eb82e08f12dd9cc29f36c77f5638426
   - name: poweroff
     image: linuxkit/poweroff:3845c4d64d47a1ea367806be5547e44594b0fa91
 trust:

src/cmd/linuxkit/vendor/github.com/moby/tool/src/moby/output.go

@@ -8,24 +8,40 @@ import (
 	"io/ioutil"
 	"os"
 	"runtime"
+	"strings"
 
 	"github.com/moby/tool/src/initrd"
 	log "github.com/sirupsen/logrus"
 )
 
-const (
+var (
-	isoBios    = "linuxkit/mkimage-iso-bios:9a51dc64a461f1cc50ba05f30a38f73f5227ac03"
+	outputImages = map[string]string{
-	isoEfi     = "linuxkit/mkimage-iso-efi:343cf1a8ac0aba7d8a1f13b7f45fa0b57ab897dc"
+		"iso-bios":    "linuxkit/mkimage-iso-bios:9a51dc64a461f1cc50ba05f30a38f73f5227ac03",
-	rawBios    = "linuxkit/mkimage-raw-bios:d90713b2dd610cf9a0f5f9d9095f8bf86f40d5c6"
+		"iso-efi":     "linuxkit/mkimage-iso-efi:343cf1a8ac0aba7d8a1f13b7f45fa0b57ab897dc",
-	rawEfi     = "linuxkit/mkimage-raw-efi:8938ffb6014543e557b624a40cce1714f30ce4b6"
+		"raw-bios":    "linuxkit/mkimage-raw-bios:d90713b2dd610cf9a0f5f9d9095f8bf86f40d5c6",
-	gcp        = "linuxkit/mkimage-gcp:e6cdcf859ab06134c0c37a64ed5f886ec8dae1a1"
+		"raw-efi":     "linuxkit/mkimage-raw-efi:8938ffb6014543e557b624a40cce1714f30ce4b6",
-	vhd        = "linuxkit/mkimage-vhd:3820219e5c350fe8ab2ec6a217272ae82f4b9242"
+		"squashfs":    "linuxkit/mkimage-squashfs:b44d00b0a336fd32c122ff32bd2b39c36a965135",
-	vmdk       = "linuxkit/mkimage-vmdk:cee81a3ed9c44ae446ef7ebff8c42c1e77b3e1b5"
+		"gcp":         "linuxkit/mkimage-gcp:e6cdcf859ab06134c0c37a64ed5f886ec8dae1a1",
-	dynamicvhd = "linuxkit/mkimage-dynamic-vhd:743ac9959fe6d3912ebd78b4fd490b117c53f1a6"
+		"qcow2-efi":   "linuxkit/mkimage-qcow2-efi:787b54906e14a56b9f1da35dcc8e46bd58435285",
-	rpi3       = "linuxkit/mkimage-rpi3:0f23c4f37cdca99281ca33ac6188e1942fa7a2b8"
+		"vhd":         "linuxkit/mkimage-vhd:3820219e5c350fe8ab2ec6a217272ae82f4b9242",
-	qcow2Efi   = "linuxkit/mkimage-qcow2-efi:787b54906e14a56b9f1da35dcc8e46bd58435285"
+		"dynamic-vhd": "linuxkit/mkimage-dynamic-vhd:743ac9959fe6d3912ebd78b4fd490b117c53f1a6",
+		"vmdk":        "linuxkit/mkimage-vmdk:cee81a3ed9c44ae446ef7ebff8c42c1e77b3e1b5",
+		"rpi3":        "linuxkit/mkimage-rpi3:0f23c4f37cdca99281ca33ac6188e1942fa7a2b8",
+	}
 )
 
+// UpdateOutputImages overwrite the docker images used to build the outputs
+// 'update' is a map where the key is the output format and the value is a LinuxKit 'mkimage' image.
+func UpdateOutputImages(update map[string]string) error {
+	for k, img := range update {
+		if _, ok := outputImages[k]; !ok {
+			return fmt.Errorf("Image format %s is not known", k)
+		}
+		outputImages[k] = img
+	}
+	return nil
+}
+
 var outFuns = map[string]func(string, io.Reader, int) error{
 	"kernel+initrd": func(base string, image io.Reader, size int) error {
 		kernel, initrd, cmdline, ucode, err := tarToInitrd(image)
@@ -49,14 +65,14 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		return nil
 	},
 	"iso-bios": func(base string, image io.Reader, size int) error {
-		err := outputIso(isoBios, base+".iso", image)
+		err := outputIso(outputImages["iso-bios"], base+".iso", image)
 		if err != nil {
 			return fmt.Errorf("Error writing iso-bios output: %v", err)
 		}
 		return nil
 	},
 	"iso-efi": func(base string, image io.Reader, size int) error {
-		err := outputIso(isoEfi, base+"-efi.iso", image)
+		err := outputIso(outputImages["iso-efi"], base+"-efi.iso", image)
 		if err != nil {
 			return fmt.Errorf("Error writing iso-efi output: %v", err)
 		}
@@ -68,7 +84,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
 		// TODO: Handle ucode
-		err = outputImg(rawBios, base+"-bios.img", kernel, initrd, cmdline)
+		err = outputImg(outputImages["raw-bios"], base+"-bios.img", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing raw-bios output: %v", err)
 		}
@@ -79,12 +95,19 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImg(rawEfi, base+"-efi.img", kernel, initrd, cmdline)
+		err = outputImg(outputImages["raw-efi"], base+"-efi.img", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing raw-efi output: %v", err)
 		}
 		return nil
 	},
+	"kernel+squashfs": func(base string, image io.Reader, size int) error {
+		err := outputKernelSquashFS(outputImages["squashfs"], base, image)
+		if err != nil {
+			return fmt.Errorf("Error writing kernel+squashfs output: %v", err)
+		}
+		return nil
+	},
 	"aws": func(base string, image io.Reader, size int) error {
 		filename := base + ".raw"
 		log.Infof("  %s", filename)
@@ -103,7 +126,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImg(gcp, base+".img.tar.gz", kernel, initrd, cmdline)
+		err = outputImg(outputImages["gcp"], base+".img.tar.gz", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing gcp output: %v", err)
 		}
@@ -114,7 +137,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImg(qcow2Efi, base+"-efi.qcow2", kernel, initrd, cmdline)
+		err = outputImg(outputImages["qcow2-efi"], base+"-efi.qcow2", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing qcow2 EFI output: %v", err)
 		}
@@ -139,7 +162,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImg(vhd, base+".vhd", kernel, initrd, cmdline)
+		err = outputImg(outputImages["vhd"], base+".vhd", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing vhd output: %v", err)
 		}
@@ -150,7 +173,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImg(dynamicvhd, base+".vhd", kernel, initrd, cmdline)
+		err = outputImg(outputImages["dynamic-vhd"], base+".vhd", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing vhd output: %v", err)
 		}
@@ -161,7 +184,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if err != nil {
 			return fmt.Errorf("Error converting to initrd: %v", err)
 		}
-		err = outputImg(vmdk, base+".vmdk", kernel, initrd, cmdline)
+		err = outputImg(outputImages["vmdk"], base+".vmdk", kernel, initrd, cmdline)
 		if err != nil {
 			return fmt.Errorf("Error writing vmdk output: %v", err)
 		}
@@ -171,7 +194,7 @@ var outFuns = map[string]func(string, io.Reader, int) error{
 		if runtime.GOARCH != "arm64" {
 			return fmt.Errorf("Raspberry Pi output currently only supported on arm64")
 		}
-		err := outputRPi3(rpi3, base+".tar", image)
+		err := outputRPi3(outputImages["rpi3"], base+".tar", image)
 		if err != nil {
 			return fmt.Errorf("Error writing rpi3 output: %v", err)
 		}
@@ -413,3 +436,57 @@ func outputKernelInitrdTarball(base string, kernel []byte, initrd []byte, cmdlin
 	}
 	return tw.Close()
 }
+
+func outputKernelSquashFS(image, base string, filesystem io.Reader) error {
+	log.Debugf("output kernel/squashfs: %s %s", image, base)
+	log.Infof("  %s-squashfs.img", base)
+
+	tr := tar.NewReader(filesystem)
+	buf := new(bytes.Buffer)
+	rootfs := tar.NewWriter(buf)
+
+	for {
+		var thdr *tar.Header
+		thdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+		switch {
+		case thdr.Name == "boot/kernel":
+			kernel, err := ioutil.ReadAll(tr)
+			if err != nil {
+				return err
+			}
+			if err := ioutil.WriteFile(base+"-kernel", kernel, os.FileMode(0644)); err != nil {
+				return err
+			}
+		case thdr.Name == "boot/cmdline":
+			cmdline, err := ioutil.ReadAll(tr)
+			if err != nil {
+				return err
+			}
+			if err := ioutil.WriteFile(base+"-cmdline", cmdline, os.FileMode(0644)); err != nil {
+				return err
+			}
+		case strings.HasPrefix(thdr.Name, "boot/"):
+			// skip the rest of boot/
+		default:
+			rootfs.WriteHeader(thdr)
+			if _, err := io.Copy(rootfs, tr); err != nil {
+				return err
+			}
+		}
+	}
+	rootfs.Close()
+
+	output, err := os.Create(base + "-squashfs.img")
+	if err != nil {
+		return err
+	}
+	defer output.Close()
+
+	return dockerRun(buf, output, true, image)
+}

src/cmd/linuxkit/vendor/github.com/moby/tool/vendor.conf

@@ -1,4 +1,4 @@
-github.com/agl/ed25519 5312a61534124124185d41f09206b9fef1d88403
+github.com/agl/ed25519 278e1ec8e8a6e017cd07577924d6766039146ced
 github.com/containerd/containerd v1.0.0
 github.com/docker/distribution 3800056b8832cf6075e78b282ac010131d8687bc
 github.com/docker/docker ba99c19b593bdb9e7b90793681fe89b0a91781ed
@@ -6,19 +6,19 @@ github.com/docker/go d30aec9fd63c35133f8f79c3412ad91a3b08be06
 github.com/docker/go-connections v0.3.0
 github.com/docker/go-units v0.3.1
 github.com/gogo/protobuf v0.5
-github.com/gorilla/mux 4c1c3952b7d9d0a061a3fa7b36fd373ba0398ebc
+github.com/gorilla/mux v1.6.1
 github.com/opencontainers/go-digest 21dfd564fd89c944783d00d069f33e3e7123c448
 github.com/opencontainers/image-spec v1.0.0
 github.com/opencontainers/runtime-spec v1.0.0
 github.com/pkg/errors v0.8.0
 github.com/sirupsen/logrus v1.0.3
 github.com/surma/gocpio fcb68777e7dc4ea43ffce871b552c0d073c17495
-github.com/theupdateframework/notary v0.6.0
+github.com/theupdateframework/notary v0.6.1
 github.com/xeipuuv/gojsonpointer 6fe8760cad3569743d51ddbb243b26f8456742dc
 github.com/xeipuuv/gojsonreference e02fc20de94c78484cd5ffb007f8af96be030a45
 github.com/xeipuuv/gojsonschema 702b404897d4364af44dc8dcabc9815947942325
 golang.org/x/crypto 573951cbe80bb6352881271bb276f48749eab6f4
 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6
 golang.org/x/sys 739734461d1c916b6c72a63d7efda2b27edb369f
-gopkg.in/yaml.v2 a3f3340b5840cee44f372bddb5880fcbc419b46a
+gopkg.in/yaml.v2 v2.2.1
 github.com/Microsoft/go-winio v0.4.1

test/cases/000_build/000_formats/011_kernel+squashfs/test.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# SUMMARY: Check that kernel+initrd output format is generated
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+NAME=check
+
+clean_up() {
+	rm -f ${NAME}*
+}
+
+trap clean_up EXIT
+
+linuxkit build -format kernel+squashfs -name "${NAME}" ../test.yml
+[ -f "${NAME}-kernel" ] || exit 1
+[ -f "${NAME}-squashfs.img" ] || exit 1
+[ -f "${NAME}-cmdline" ] || exit 1
+
+exit 0

test/cases/010_platforms/000_qemu/005_run_kernel+squashfs/test.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# SUMMARY: Check that the kernel+squashfs image boots in qemu
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+NAME=qemu-squashfs
+
+clean_up() {
+	rm -rf ${NAME}-*
+}
+trap clean_up EXIT
+
+linuxkit build -format kernel+squashfs -name "${NAME}" test.yml
+[ -f "${NAME}-kernel" ] || exit 1
+[ -f "${NAME}-squashfs.img" ] || exit 1
+[ -f "${NAME}-cmdline" ]|| exit 1
+linuxkit run qemu -squashfs "${NAME}" | grep -q "Welcome to LinuxKit"
+
+exit 0

test/cases/010_platforms/000_qemu/005_run_kernel+squashfs/test.yml

@@ -0,0 +1,13 @@
+kernel:
+  image: linuxkit/kernel:4.14.34
+  cmdline: "console=ttyS0 console=ttyAMA0"
+init:
+  - linuxkit/init:v0.3
+  - linuxkit/runc:v0.3
+onboot:
+  - name: poweroff
+    image: linuxkit/poweroff:5740687bf0a6a0480922c0f59b3a4ca77c866cae
+    command: ["/bin/sh", "/poweroff.sh", "10"]
+trust:
+  org:
+    - linuxkit

test/cases/010_platforms/010_hyperkit/005_run_kernel+squashfs/test.exp

@@ -0,0 +1,31 @@
+#!/usr/bin/env expect
+spawn linuxkit run hyperkit -squashfs hyperkit-squashfs
+set pid [exp_pid]
+set timeout 30
+
+expect {
+    timeout {
+        puts "FAILED boot"
+        exec kill -9 $pid
+        exit 1
+    }
+    "Welcome to LinuxKit" {
+        puts "SUCCESS boot"
+    }
+}
+expect {
+    timeout {
+        puts "FAILED poweroff"
+        exec kill -9 $pid
+        exit 1
+    }
+    "Power down" {
+        puts "SUCCESS poweroff"
+    }
+    eof {
+        puts "SUCCESS poweroff"
+    }
+}
+set waitval [wait -i $spawn_id]
+set exval [lindex $waitval 3]
+exit $exval

test/cases/010_platforms/010_hyperkit/005_run_kernel+squashfs/test.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# SUMMARY: Check that the kernel+squashfs image boots on hyperkit
+# LABELS:
+
+set -e
+
+# Source libraries. Uncomment if needed/defined
+#. "${RT_LIB}"
+. "${RT_PROJECT_ROOT}/_lib/lib.sh"
+
+NAME=hyperkit-squashfs
+
+clean_up() {
+	rm -rf ${NAME}-*
+}
+trap clean_up EXIT
+
+linuxkit build -format kernel+squashfs -name "${NAME}" test.yml
+[ -f "${NAME}-kernel" ] || exit 1
+[ -f "${NAME}-squashfs.img" ] || exit 1
+[ -f "${NAME}-cmdline" ]|| exit 1
+./test.exp
+
+exit 0

test/cases/010_platforms/010_hyperkit/005_run_kernel+squashfs/test.yml

@@ -0,0 +1,13 @@
+kernel:
+  image: linuxkit/kernel:4.14.34
+  cmdline: "console=ttyS0"
+init:
+  - linuxkit/init:v0.3
+  - linuxkit/runc:v0.3
+onboot:
+  - name: poweroff
+    image: linuxkit/poweroff:5740687bf0a6a0480922c0f59b3a4ca77c866cae
+    command: ["/bin/sh", "/poweroff.sh", "10"]
+trust:
+  org:
+    - linuxkit

tools/mkimage-squashfs/Dockerfile

@@ -0,0 +1,12 @@
+FROM linuxkit/alpine:f3cd219615428b2bd943411723eb28875275fae7
+
+RUN \
+  apk update && apk upgrade && \
+  apk add --no-cache \
+  libarchive-tools \
+  squashfs-tools \
+  && true
+
+COPY . .
+
+ENTRYPOINT [ "/make-squashfs" ]

tools/mkimage-squashfs/build.yml

@@ -0,0 +1 @@
+image: mkimage-squashfs

tools/mkimage-squashfs/make-squashfs

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -e
+
+mkdir -p /tmp/rootfs
+cd /tmp/rootfs
+
+# input is a tarball of filesystem on stdin with the root filesytem
+# output is a squashfs image on stdout
+
+# extract. BSD tar auto recognises compression, unlike GNU tar
+# only if stdin is a tty, if so need files volume mounted...
+[ -t 0 ] || bsdtar xzf -
+cd /tmp
+
+# we want everything except the final result to stderr
+(
+    exec 1>&2;
+
+    mksquashfs rootfs ./rootfs.img
+)
+cat rootfs.img