vendor: Update hyperkit go bindings

This contains a small fix to the disk binadings and allows
booting with a kernel alone (no initrd).

Signed-off-by: Rolf Neugebauer <rolf.neugebauer@docker.com>

src/cmd/linuxkit/vendor.conf

@@ -23,7 +23,7 @@ github.com/gorilla/mux 4c1c3952b7d9d0a061a3fa7b36fd373ba0398ebc
 github.com/jmespath/go-jmespath bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d
 github.com/mitchellh/go-ps 4fdf99ab29366514c69ccccddab5dc58b8d84062
 github.com/moby/datakit 97b3d230535397a813323902c23751e176481a86
-github.com/moby/hyperkit a285521725f44f3d10ca1042c2c07d3a6e24bed8
+github.com/moby/hyperkit d65b09c1c28a2bfb6a976c86ecd885d2ee4c71d3
 # When updating also:
 # curl -fsSL -o src/cmd/linuxkit/build.go https://raw.githubusercontent.com/moby/tool/«hash»/cmd/moby/build.go
 github.com/moby/tool 3dbad3b7daffd631d036493a1e883608206d2e03

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/README.md

@@ -1,10 +1,10 @@
 ## [HyperKit](http://github.com/moby/hyperkit)
 
-![Build Status OSX](https://circleci.com/gh/moby/hyperkit.svg?style=shield&circle-token=cf8379b302eab2bbf33821cafe164dbefb71982d)
+![Build Status macOS](https://circleci.com/gh/moby/hyperkit.svg?style=shield&circle-token=cf8379b302eab2bbf33821cafe164dbefb71982d)
 
 *HyperKit* is a toolkit for embedding hypervisor capabilities in your application. It includes a complete hypervisor, based on [xhyve](https://github.com/mist64/xhyve)/[bhyve](http://bhyve.org), which is optimized for lightweight virtual machines and container deployment.  It is designed to be interfaced with higher-level components such as the [VPNKit](https://github.com/moby/vpnkit) and [DataKit](https://github.com/moby/datakit).
 
-HyperKit currently only supports Mac OS X using the [Hypervisor.framework](https://developer.apple.com/library/mac/documentation/DriversKernelHardware/Reference/Hypervisor/index.html). It is a core component of Docker For Mac.
+HyperKit currently only supports macOS using the [Hypervisor.framework](https://developer.apple.com/library/mac/documentation/DriversKernelHardware/Reference/Hypervisor/index.html). It is a core component of Docker For Mac.
 
 
 ## Requirements

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/go/disk.go

@@ -65,7 +65,7 @@ func GetDiskFormat(path string) DiskFormat {
 	case ".raw", ".img":
 		return DiskFormatRaw
 	default:
-		log.Debugf("Unknown disk extension %q, will use raw format", path)
+		log.Debugf("hyperkit: Unknown disk extension %q, will use raw format", path)
 		return DiskFormatRaw
 	}
 }
@@ -96,7 +96,7 @@ func NewDisk(spec string, size int) (Disk, error) {
 func exists(d Disk) bool {
 	_, err := os.Stat(d.GetPath())
 	if err != nil && !os.IsNotExist(err) {
-		log.Debugf("cannot stat %q: %v", d, err)
+		log.Debugf("hyperkit: cannot stat %q: %v", d, err)
 	}
 	return err == nil
 }
@@ -114,7 +114,7 @@ func ensure(d Disk) error {
 		return d.resize()
 	}
 	if d.GetSize() < current {
-		log.Errorf("Cannot safely shrink %q from %dMiB to %dMiB", d, current, d.GetSize())
+		log.Errorf("hyperkit: Cannot safely shrink %q from %dMiB to %dMiB", d, current, d.GetSize())
 	}
 	return nil
 }
@@ -199,7 +199,7 @@ func (d *RawDisk) Ensure() error {
 
 // Create a disk.
 func (d *RawDisk) create() error {
-	log.Infof("Create %q", d)
+	log.Infof("hyperkit: Create %q", d)
 	f, err := os.Create(d.Path)
 	if err != nil {
 		return err
@@ -225,7 +225,7 @@ func (d *RawDisk) resize() error {
 	if err != nil {
 		return fmt.Errorf("Cannot resize %q: %v", d, err)
 	}
-	log.Infof("Resize %q from %vMiB to %vMiB", d, s, d.GetSize())
+	log.Infof("hyperkit: Resize %q from %vMiB to %vMiB", d, s, d.GetSize())
 	// APFS exhibits a weird behavior wrt sparse files: we cannot
 	// create (or grow) them "too fast": there's a limit,
 	// apparently related to the available disk space.  However,
@@ -241,7 +241,7 @@ func (d *RawDisk) resize() error {
 			return fmt.Errorf("Cannot resize %q to %vMiB: %v", d, s, err)
 		}
 	}
-	log.Infof("Resized %q to %vMiB", d, d.GetSize())
+	log.Infof("hyperkit: Resized %q to %vMiB", d, d.GetSize())
 	return nil
 }
 
@@ -314,7 +314,7 @@ func (d *QcowDisk) QcowTool(verb string, args ...string) *exec.Cmd {
 func run(cmd *exec.Cmd) (string, error) {
 	buf, err := cmd.CombinedOutput()
 	out := string(buf)
-	log.Debugf("ran %v: out=%q, err=%v", cmd.Args, out, err)
+	log.Debugf("hyperkit: ran %v: out=%q, err=%v", cmd.Args, out, err)
 	return out, err
 }
 
@@ -326,18 +326,18 @@ func (d *QcowDisk) Exists() bool {
 // Ensure creates the disk image if needed, and resizes it if needed.
 func (d *QcowDisk) Ensure() error {
 	if d.Trim {
-		log.Infof("%v: TRIM is enabled; recycling thread will keep %v sectors free and will compact after %v more sectors are free",
+		log.Infof("hyperkit: %v: TRIM is enabled; recycling thread will keep %v sectors free and will compact after %v more sectors are free",
 			d, d.KeepErased, d.CompactAfter)
 	}
 	if d.RuntimeAsserts {
-		log.Warnf("%v: Expensive runtime checks are enabled", d)
+		log.Warnf("hyperkit: %v: Expensive runtime checks are enabled", d)
 	}
 	return ensure(d)
 }
 
 // Create a disk with the given size in MiB
 func (d *QcowDisk) create() error {
-	log.Infof("Create %q", d)
+	log.Infof("hyperkit: Create %q", d)
 	_, err := run(d.QcowTool("create", "--size", fmt.Sprintf("%dMiB", d.Size)))
 	return err
 }
@@ -368,14 +368,14 @@ func (d *QcowDisk) sizeString() string {
 
 // Resize the virtual size of the disk
 func (d *QcowDisk) resize() error {
-	log.Infof("Resize %q from %v to %dMiB", d, d.sizeString(), d.GetSize())
+	log.Infof("hyperkit: Resize %q from %v to %dMiB", d, d.sizeString(), d.GetSize())
 	_, err := run(d.QcowTool("resize", "--size", fmt.Sprintf("%dMiB", d.Size)))
 	return err
 }
 
 // compact the disk to shrink the physical size.
 func (d *QcowDisk) compact() error {
-	log.Infof("Compact: %q... (%v)", d, d.sizeString())
+	log.Infof("hyperkit: Compact: %q... (%v)", d, d.sizeString())
 	cmd := d.QcowTool("compact")
 	if _, err := run(cmd); err != nil {
 		if err.(*exec.ExitError) != nil {
@@ -383,7 +383,7 @@ func (d *QcowDisk) compact() error {
 		}
 		return err
 	}
-	log.Infof("Compact: %q: done (%v)", d, d.sizeString())
+	log.Infof("hyperkit: Compact: %q: done (%v)", d, d.sizeString())
 	return nil
 }
 
@@ -402,14 +402,14 @@ func (d *QcowDisk) check() error {
 // Stop cleans up this disk when we are quitting.
 func (d *QcowDisk) Stop() error {
 	if !d.Trim && d.CompactAfter == 0 {
-		log.Infof("TRIM is enabled but auto-compaction disabled: compacting %q now", d)
+		log.Infof("hyperkit: TRIM is enabled but auto-compaction disabled: compacting %q now", d)
 		if err := d.compact(); err != nil {
 			return fmt.Errorf("Failed to compact %q: %v", d, err)
 		}
 		if err := d.check(); err != nil {
 			return fmt.Errorf("Post-compact disk integrity check of %q failed: %v", d, err)
 		}
-		log.Infof("Post-compact disk integrity check of %q successful", d)
+		log.Infof("hyperkit: Post-compact disk integrity check of %q successful", d)
 	}
 	return nil
 }

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/go/hyperkit.go

@@ -44,6 +44,8 @@ const (
 	ConsoleStdio = iota
 	// ConsoleFile configures console to a tty and output to a file
 	ConsoleFile
+	// ConsoleLog configures console to a tty and sends its contents to the logs
+	ConsoleLog
 
 	defaultVPNKitSock = "Library/Containers/com.docker.docker/Data/s50"
 
@@ -117,8 +119,7 @@ type HyperKit struct {
 	// Memory is the amount of megabytes of memory for the VM.
 	Memory int `json:"memory"`
 
-	// Console defines where the console of the VM should be
-	// connected to. ConsoleStdio and ConsoleFile are supported.
+	// Console defines where the console of the VM should be connected to.
 	Console int `json:"console"`
 
 	// Below here are internal members, but they are exported so
@@ -199,11 +200,15 @@ func (h *HyperKit) check() error {
 	log.Debugf("hyperkit: check %#v", h)
 	var err error
 	// Sanity checks on configuration
-	if h.Console == ConsoleFile && h.StateDir == "" {
-		return fmt.Errorf("If ConsoleFile is set, StateDir must be specified")
-	}
-	if h.Console == ConsoleStdio && !isTerminal(os.Stdout) && h.StateDir == "" {
-		return fmt.Errorf("If ConsoleStdio is set but stdio is not a terminal, StateDir must be specified")
+	switch h.Console {
+	case ConsoleFile, ConsoleLog:
+		if h.StateDir == "" {
+			return fmt.Errorf("If ConsoleFile or ConsoleLog is set, StateDir must be specified")
+		}
+	case ConsoleStdio:
+		if !isTerminal(os.Stdout) && h.StateDir == "" {
+			return fmt.Errorf("If ConsoleStdio is set but stdio is not a terminal, StateDir must be specified")
+		}
 	}
 	for _, image := range h.ISOImages {
 		if _, err = os.Stat(image); os.IsNotExist(err) {
@@ -226,8 +231,10 @@ func (h *HyperKit) check() error {
 		if _, err = os.Stat(h.Kernel); os.IsNotExist(err) {
 			return fmt.Errorf("Kernel %s does not exist", h.Kernel)
 		}
-		if _, err = os.Stat(h.Initrd); os.IsNotExist(err) {
-			return fmt.Errorf("initrd %s does not exist", h.Initrd)
+		if h.Initrd != "" {
+			if _, err = os.Stat(h.Initrd); os.IsNotExist(err) {
+				return fmt.Errorf("initrd %s does not exist", h.Initrd)
+			}
 		}
 	} else {
 		if _, err = os.Stat(h.Bootrom); os.IsNotExist(err) {
@@ -420,10 +427,20 @@ func (h *HyperKit) buildArgs(cmdline string) {
 		nextSlot++
 	}
 
-	if h.Console == ConsoleStdio && isTerminal(os.Stdout) {
-		a = append(a, "-l", "com1,stdio")
-	} else if h.StateDir != "" {
-		a = append(a, "-l", fmt.Sprintf("com1,autopty=%s/tty,log=%s/console-ring", h.StateDir, h.StateDir))
+	// -l: LPC device configuration.
+	{
+		cfg := "com1"
+		if h.Console == ConsoleStdio && isTerminal(os.Stdout) {
+			cfg += fmt.Sprintf(",stdio")
+		} else {
+			cfg += fmt.Sprintf(",autopty=%s/tty", h.StateDir)
+		}
+		if h.Console == ConsoleLog {
+			cfg += fmt.Sprintf(",asl")
+		} else {
+			cfg += fmt.Sprintf(",log=%s/console-ring", h.StateDir)
+		}
+		a = append(a, "-l", cfg)
 	}
 
 	if h.Bootrom == "" {
@@ -440,6 +457,22 @@ func (h *HyperKit) buildArgs(cmdline string) {
 	log.Debugf("hyperkit: CmdLine: %#v", h.CmdLine)
 }
 
+// openTTY opens the tty files for reading, and returns it.
+func (h *HyperKit) openTTY() *os.File {
+	path := fmt.Sprintf("%s/tty", h.StateDir)
+	for {
+		if res, err := os.OpenFile(path, os.O_RDONLY, 0); err != nil {
+			log.Infof("hyperkit: openTTY: %v, retrying", err)
+			time.Sleep(10 * time.Millisecond)
+		} else {
+			log.Infof("hyperkit: openTTY: got %v", path)
+			saneTerminal(res)
+			setRaw(res)
+			return res
+		}
+	}
+}
+
 // execute forges the command to run hyperkit, runs and returns it.
 // It also plumbs stdin/stdout/stderr.
 func (h *HyperKit) execute() (*exec.Cmd, error) {
@@ -469,26 +502,13 @@ func (h *HyperKit) execute() (*exec.Cmd, error) {
 			cmd.Stderr = os.Stderr
 		} else {
 			go func() {
-				ttyPath := fmt.Sprintf("%s/tty", h.StateDir)
-				var tty *os.File
-				for {
-					var err error
-					tty, err = os.OpenFile(ttyPath, os.O_RDONLY, 0)
-					if err == nil {
-						break
-					}
-					time.Sleep(10 * time.Millisecond)
-				}
-				saneTerminal(tty)
-				setRaw(tty)
+				tty := h.openTTY()
+				defer tty.Close()
 				io.Copy(os.Stdout, tty)
-				tty.Close()
 			}()
 		}
 	} else if log != nil {
 		log.Debugf("hyperkit: Redirecting stdout/stderr to logger")
-		stdoutChan := make(chan string)
-		stderrChan := make(chan string)
 		stdout, err := cmd.StdoutPipe()
 		if err != nil {
 			return nil, err
@@ -497,22 +517,8 @@ func (h *HyperKit) execute() (*exec.Cmd, error) {
 		if err != nil {
 			return nil, err
 		}
-		stream(stdout, stdoutChan)
-		stream(stderr, stderrChan)
-
-		done := make(chan struct{})
-		go func() {
-			for {
-				select {
-				case stderrl := <-stderrChan:
-					log.Infof("%s", stderrl)
-				case stdoutl := <-stdoutChan:
-					log.Infof("%s", stdoutl)
-				case <-done:
-					return
-				}
-			}
-		}()
+		go logStream(stdout, "stdout")
+		go logStream(stderr, "stderr")
 	}
 
 	log.Debugf("hyperkit: Starting %#v", cmd)
@@ -546,18 +552,18 @@ func (h *HyperKit) writeState() error {
 	return ioutil.WriteFile(filepath.Join(h.StateDir, jsonFile), []byte(s), 0644)
 }
 
-func stream(r io.ReadCloser, dest chan<- string) {
-	go func() {
-		defer r.Close()
-		reader := bufio.NewReader(r)
-		for {
-			line, err := reader.ReadString('\n')
-			if err != nil {
-				return
-			}
-			dest <- line
+// logStream redirects a file to the logs.
+func logStream(r io.ReadCloser, name string) {
+	defer r.Close()
+	reader := bufio.NewReader(r)
+	for {
+		line, err := reader.ReadString('\n')
+		if err != nil {
+			log.Warnf("hyperkit: failed to read %v: %v", name, err)
+			break
 		}
-	}()
+		log.Infof("hyperkit: %v: %v", name, line)
+	}
 }
 
 // checkHyperKit tries to find and/or validate the path of hyperkit

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/go/pty_util_fallback.go

@@ -7,17 +7,17 @@ import (
 )
 
 func saneTerminal(f *os.File) error {
-	log.Fatalf("Function not supported on your OS")
+	log.Fatalf("hyperkit: Function not supported on your OS")
 	return nil
 }
 
 func setRaw(f *os.File) error {
-	log.Fatalf("Function not supported on your OS")
+	log.Fatalf("hyperkit: Function not supported on your OS")
 	return nil
 }
 
 // isTerminal checks if the provided file is a terminal
 func isTerminal(f *os.File) bool {
-	log.Fatalf("Function not supported on your OS")
+	log.Fatalf("hyperkit: Function not supported on your OS")
 	return false
 }

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/include/xhyve/log.h

@@ -0,0 +1,7 @@
+#pragma once
+
+/* Initialize ASL logger and local buffer. */
+void log_init(void);
+
+/* Send one character to the logger: wait for full lines before actually sending. */
+void log_put(uint8_t _c);

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/lib/block_if.c

@@ -257,27 +257,40 @@ block_delete(struct blockif_ctxt *bc, off_t offset, off_t len)
 
 		size_t aligned_offset = ALIGNUP(fp_offset, bc->bc_delete_alignment);
 		if (aligned_offset != fp_offset) {
-			size_t len_to_zero = aligned_offset - fp_offset;
-			assert(len_to_zero <= bc->bc_delete_alignment);
-			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf, len_to_zero, (off_t) fp_offset);
+			size_t len_to_zero = MIN(fp_length, aligned_offset - fp_offset);
+			assert(len_to_zero < bc->bc_delete_alignment);
+			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf,
+				len_to_zero, (off_t)fp_offset);
 			if (written == -1) goto out;
 			fp_offset += len_to_zero;
 			fp_length -= len_to_zero;
 		}
 		size_t aligned_length = ALIGNDOWN(fp_length, bc->bc_delete_alignment);
-		if (aligned_length != fp_length) {
-			size_t len_to_zero = fp_length - aligned_length;
-			assert(len_to_zero <= bc->bc_delete_alignment);
-			fp_length -= len_to_zero;
-			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf, len_to_zero, (off_t) (fp_offset + fp_length));
+		if (aligned_length >= bc->bc_delete_alignment) {
+			assert(fp_offset % bc->bc_delete_alignment == 0);
+			struct fpunchhole arg = {
+				.fp_flags = 0,
+				.reserved = 0,
+				.fp_offset = (off_t)fp_offset,
+				.fp_length = (off_t)aligned_length
+			};
+			int punched = fcntl(bc->bc_fd, F_PUNCHHOLE, &arg);
+			if (punched == -1) goto out;
+			fp_offset += aligned_length;
+			fp_length -= aligned_length;
+		}
+		if (fp_length > 0) {
+			assert(fp_length < bc->bc_delete_alignment);
+			assert(fp_offset % bc->bc_delete_alignment == 0);
+			ssize_t written = pwrite(bc->bc_fd, bc->bc_delete_zero_buf,
+				fp_length, (off_t)fp_offset);
 			if (written == -1) goto out;
 		}
-		struct fpunchhole arg = { .fp_flags = 0, .reserved = 0, .fp_offset = (off_t) fp_offset, .fp_length = (off_t) fp_length };
-		ret = fcntl(bc->bc_fd, F_PUNCHHOLE, &arg);
-		goto out;
+		ret = 0;
 	}
-#endif
+#else
 	errno = EOPNOTSUPP;
+#endif
 out:
 	HYPERKIT_BLOCK_DELETE_DONE(offset, ret);
 	return ret;

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/lib/log.c

@@ -0,0 +1,50 @@
+#include <asl.h>
+#include <pwd.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <time.h>
+
+#include <SystemConfiguration/SystemConfiguration.h>
+
+#include <xhyve/log.h>
+
+static aslclient log_client = NULL;
+static aslmsg log_msg = NULL;
+
+static unsigned char buf[4096];
+/* Index of the _next_ character to insert in the buffer. */
+static size_t buf_idx = 0;
+
+/* asl is deprecated in favor of os_log starting with macOS 10.12.  */
+#pragma GCC diagnostic ignored "-Wdeprecated-declarations"
+
+/* Initialize ASL logger and local buffer. */
+void log_init(void)
+{
+	log_client = asl_open(NULL, NULL, 0);
+	log_msg = asl_new(ASL_TYPE_MSG);
+}
+
+
+/* Send the content of the buffer to the logger. */
+static void log_flush(void)
+{
+	buf[buf_idx] = 0;
+	asl_log(log_client, log_msg, ASL_LEVEL_NOTICE, "%s", buf);
+	buf_idx = 0;
+}
+
+
+/* Send one character to the logger: wait for full lines before actually sending. */
+void log_put(uint8_t c)
+{
+	if ((c == '\n') || (c == 0)) {
+		log_flush();
+	} else {
+		if (buf_idx + 2 >= sizeof(buf)) {
+			log_flush();
+		}
+		buf[buf_idx] = c;
+		++buf_idx;
+	}
+}

src/cmd/linuxkit/vendor/github.com/moby/hyperkit/src/lib/uart_emul.c

@@ -41,13 +41,14 @@
 #include <assert.h>
 #include <errno.h>
 #include <sys/mman.h>
+#include <xhyve/log.h>
 #include <xhyve/support/ns16550.h>
 #include <xhyve/mevent.h>
 #include <xhyve/uart_emul.h>
 
-#define	COM1_BASE      	0x3F8
+#define	COM1_BASE	0x3F8
 #define COM1_IRQ	4
-#define	COM2_BASE      	0x2F8
+#define	COM2_BASE	0x2F8
 #define COM2_IRQ	3
 
 #define	DEFAULT_RCLK	1843200
@@ -91,7 +92,7 @@ struct fifo {
 struct ttyfd {
 	bool	opened;
 	int	fd;		/* tty device file descriptor */
-	int 	sfd;
+	int	sfd;
 	char *name; /* slave pty name when using autopty*/
 	struct termios tio_orig, tio_new;    /* I/O Terminals */
 };
@@ -121,6 +122,7 @@ struct uart_softc {
 
 	struct ttyfd tty;
 	struct log log;
+	bool	asl;		/* Output to Apple logger. */
 	bool	thre_int_pending;	/* THRE interrupt pending */
 
 	void	*arg;
@@ -427,7 +429,7 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 		}
 	}
 
-        switch (offset) {
+	switch (offset) {
 	case REG_DATA:
 		if (sc->mcr & MCR_LOOPBACK) {
 			if (rxfifo_putchar(sc, value) != 0)
@@ -436,6 +438,8 @@ uart_write(struct uart_softc *sc, int offset, uint8_t value)
 			ttywrite(&sc->tty, value);
 			if (sc->log.ring)
 				ringwrite(&sc->log, value);
+			if (sc->asl)
+				log_put(value);
 		} /* else drop on floor */
 		sc->thre_int_pending = true;
 		break;
@@ -681,15 +685,15 @@ uart_tty_backend(struct uart_softc *sc, const char *backend)
 static char *
 copy_up_to_comma(const char *from)
 {
-        char *comma = strchr(from, ',');
-        char *tmp = NULL;
-        if (comma == NULL) {
-                tmp = strdup(from); /* rest of string */
-        } else {
-                ptrdiff_t length = comma - from;
-                tmp = strndup(from, (size_t)length);
-        }
-        return tmp;
+	char *comma = strchr(from, ',');
+	char *tmp = NULL;
+	if (comma == NULL) {
+		tmp = strdup(from); /* rest of string */
+	} else {
+		ptrdiff_t length = comma - from;
+		tmp = strndup(from, (size_t)length);
+	}
+	return tmp;
 }
 
 int
@@ -773,6 +777,10 @@ uart_set_backend(struct uart_softc *sc, const char *backend, const char *devname
 			if (uart_mapring(sc, logname) == -1) {
 				goto err;
 			}
+		} else if (strcmp("asl", backend) == 0) {
+			sc->asl = true;
+			log_init();
+			retval = 0;
 		} else if (uart_tty_backend(sc, backend) == 0) {
 			retval = 0;
 		} else {