Merge pull request #1891 from MagnusS/add-9p-linuxkit

Add -start-vpnkit flag to linuxkit run hyperkit

src/cmd/linuxkit/run_hyperkit.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 
 	log "github.com/Sirupsen/logrus"
@@ -33,6 +34,7 @@ func runHyperKit(args []string) {
 	ipStr := flags.String("ip", "", "IP address for the VM")
 	state := flags.String("state", "", "Path to directory to keep VM state in")
 	vsockports := flags.String("vsock-ports", "", "List of vsock ports to forward from the guest on startup (comma separated). A unix domain socket for each port will be created in the state directory")
+	startVPNKit := flags.Bool("start-vpnkit", false, "Launch a new VPNKit instead of reusing the instance from Docker for Mac. The new instance will be on a separate internal network. This enables IP port forwarding from the host to the guest if the guest supports it.")
 
 	if err := flags.Parse(args); err != nil {
 		log.Fatal("Unable to parse args")
@@ -101,7 +103,30 @@ func runHyperKit(args []string) {
 		*disk = filepath.Join(*state, "disk.img")
 	}
 
-	h, err := hyperkit.New(*hyperkitPath, "auto", *state)
+	vpnKitEthernetSocket := "auto"
+	var vpnKitPortSocket string
+	var vpnKitProcess *os.Process
+
+	// Launch new VPNKit if needed
+	if *startVPNKit {
+		vpnKitEthernetSocket = filepath.Join(*state, "vpnkit_eth.sock")
+		vpnKitPortSocket = filepath.Join(*state, "vpnkit_port.sock")
+		vsockSocket := filepath.Join(*state, "connect")
+		vpnKitProcess, err = launchVPNKit(vpnKitEthernetSocket, vsockSocket, vpnKitPortSocket)
+		if err != nil {
+			log.Fatalln("Unable to start vpnkit: ", err)
+		}
+		defer func() {
+			if vpnKitProcess != nil {
+				err := vpnKitProcess.Kill()
+				if err != nil {
+					log.Println(err)
+				}
+			}
+		}()
+	}
+
+	h, err := hyperkit.New(*hyperkitPath, vpnKitEthernetSocket, *state)
 	if err != nil {
 		log.Fatalln("Error creating hyperkit: ", err)
 	}
@@ -121,8 +146,74 @@ func runHyperKit(args []string) {
 	h.Memory = *mem
 	h.DiskSize = diskSz
 
+	// Add 9p and vsock for port forwarding if VPNKit is launched automatically
+	if *startVPNKit {
+		// The guest will use this 9P mount to configure which ports to forward
+		h.Sockets9P = []hyperkit.Socket9P{{Path: vpnKitPortSocket, Tag: "port"}}
+		// VSOCK port 62373 is used to pass traffic from host->guest
+		h.VSockPorts = append(h.VSockPorts, 62373)
+	}
+
 	err = h.Run(string(cmdline))
 	if err != nil {
 		log.Fatalf("Cannot run hyperkit: %v", err)
 	}
 }
+
+// createListenSocket creates a new unix domain socket and returns the open file
+func createListenSocket(path string) (*os.File, error) {
+	os.Remove(path)
+	conn, err := net.ListenUnix("unix", &net.UnixAddr{Name: path, Net: "unix"})
+	if err != nil {
+		return nil, fmt.Errorf("unable to create socket: %v", err)
+	}
+	f, err := conn.File()
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+// launchVPNKit starts a new instance of VPNKit. Ethernet socket and port socket
+// will be created and passed to VPNKit. The VSOCK socket should be created
+// by HyperKit when it starts.
+func launchVPNKit(etherSock string, vsockSock string, portSock string) (*os.Process, error) {
+	var err error
+
+	vpnKitPath, err := exec.LookPath("vpnkit")
+	if err != nil {
+		return nil, fmt.Errorf("Unable to find vpnkit binary")
+	}
+
+	etherFile, err := createListenSocket(etherSock)
+	if err != nil {
+		return nil, err
+	}
+
+	portFile, err := createListenSocket(portSock)
+	if err != nil {
+		return nil, err
+	}
+
+	cmd := exec.Command(vpnKitPath,
+		"--ethernet", "fd:3",
+		"--vsock-path", vsockSock,
+		"--port", "fd:4")
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, etherFile)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, portFile)
+
+	cmd.Env = os.Environ() // pass env for DEBUG
+
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	go cmd.Wait() // run in background
+
+	return cmd.Process, nil
+}

vendor.conf

@@ -5,7 +5,7 @@ github.com/docker/infrakit cb420e3e50ea60afe58538b1d3cab1cb14059433
 github.com/golang/protobuf c9c7427a2a70d2eb3bafa0ab2dc163e45f143317
 github.com/googleapis/gax-go 8c5154c0fe5bf18cf649634d4c6df50897a32751
 github.com/mitchellh/go-ps 4fdf99ab29366514c69ccccddab5dc58b8d84062
-github.com/moby/hyperkit fa78d9472a7d98e393233fd61ad5e95adc8c6912
+github.com/moby/hyperkit 95a60c3e75ac73a1cfd73bacdb5de94a737b6095
 github.com/packethost/packngo 91d54000aa56874149d348a884ba083c41d38091
 github.com/rneugeba/iso9660wrap 4606f848a055435cdef85305960b0e1bb788d506
 github.com/satori/go.uuid b061729afc07e77a8aa4fad0a2fd840958f1942a

vendor/github.com/moby/hyperkit/README.md

@@ -38,7 +38,7 @@ via `brew` and using `opam` to install the appropriate libraries:
     $ brew install opam libev
     $ opam init
     $ eval `opam config env`
-    $ opam install uri qcow.0.9.5 mirage-block-unix.2.7.0 conf-libev logs fmt mirage-unix
+    $ opam install uri qcow.0.10.0 qcow-tool mirage-block-unix.2.7.0 conf-libev logs fmt mirage-unix prometheus-app
 
 Notes:
 

vendor/github.com/moby/hyperkit/go/hyperkit.go

@@ -63,6 +63,12 @@ var defaultHyperKits = []string{"hyperkit",
 	"/Applications/Docker.app/Contents/Resources/bin/hyperkit",
 	"/Applications/Docker.app/Contents/MacOS/com.docker.hyperkit"}
 
+// Socket9P contains a unix domain socket path and 9p tag
+type Socket9P struct {
+	Path string `json:"path"`
+	Tag  string `json:"tag"`
+}
+
 // HyperKit contains the configuration of the hyperkit VM
 type HyperKit struct {
 	// HyperKit is the path to the hyperkit binary
@@ -86,6 +92,9 @@ type HyperKit struct {
 	// VSock guest CID
 	VSockGuestCID int `json:"vsock_guest_cid"`
 
+	// 9P sockets
+	Sockets9P []Socket9P `json:"9p_sockets"`
+
 	// Kernel is the path to the kernel image to boot
 	Kernel string `json:"kernel"`
 	// Initrd is the path to the initial ramdisk to boot off
@@ -367,32 +376,49 @@ func (h *HyperKit) buildArgs(cmdline string) {
 	a = append(a, "-m", fmt.Sprintf("%dM", h.Memory))
 
 	a = append(a, "-s", "0:0,hostbridge")
+	a = append(a, "-s", "31,lpc")
+
+	nextSlot := 1
+
 	if h.VPNKitSock != "" {
 		if h.VPNKitKey == "" {
-			a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s", h.VPNKitSock))
+			a = append(a, "-s", fmt.Sprintf("%d:0,virtio-vpnkit,path=%s", nextSlot, h.VPNKitSock))
 		} else {
-			a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s,uuid=%s", h.VPNKitSock, h.VPNKitKey))
+			a = append(a, "-s", fmt.Sprintf("%d:0,virtio-vpnkit,path=%s,uuid=%s", nextSlot, h.VPNKitSock, h.VPNKitKey))
 		}
+		nextSlot++
 	}
+
 	if h.UUID != "" {
 		a = append(a, "-U", h.UUID)
 	}
+
 	if h.DiskImage != "" {
-		a = append(a, "-s", fmt.Sprintf("2:0,virtio-blk,%s", h.DiskImage))
+		a = append(a, "-s", fmt.Sprintf("%d:0,virtio-blk,%s", nextSlot, h.DiskImage))
+		nextSlot++
 	}
+
 	if h.VSock {
-		l := fmt.Sprintf("3,virtio-sock,guest_cid=%d,path=%s", h.VSockGuestCID, h.StateDir)
+		l := fmt.Sprintf("%d,virtio-sock,guest_cid=%d,path=%s", nextSlot, h.VSockGuestCID, h.StateDir)
 		if len(h.VSockPorts) > 0 {
 			l = fmt.Sprintf("%s,guest_forwards=%s", l, intArrayToString(h.VSockPorts, ";"))
 		}
 		a = append(a, "-s", l)
-	}
-	if h.ISOImage != "" {
-		a = append(a, "-s", fmt.Sprintf("4,ahci-cd,%s", h.ISOImage))
+		nextSlot++
 	}
 
-	a = append(a, "-s", "10,virtio-rnd")
-	a = append(a, "-s", "31,lpc")
+	if h.ISOImage != "" {
+		a = append(a, "-s", fmt.Sprintf("%d,ahci-cd,%s", nextSlot, h.ISOImage))
+		nextSlot++
+	}
+
+	a = append(a, "-s", fmt.Sprintf("%d,virtio-rnd", nextSlot))
+	nextSlot++
+
+	for _, p := range h.Sockets9P {
+		a = append(a, "-s", fmt.Sprintf("%d,virtio-9p,path=%s,tag=%s", nextSlot, p.Path, p.Tag))
+		nextSlot++
+	}
 
 	if h.Console == ConsoleFile {
 		a = append(a, "-l", fmt.Sprintf("com1,autopty=%s/tty,log=%s/console-ring", h.StateDir, h.StateDir))

vendor/github.com/moby/hyperkit/src/lib/block_if.c

@@ -563,6 +563,7 @@ blockif_open(const char *optstr, const char *ident)
 
 #ifdef HAVE_OCAML_QCOW
 	char *mirage_qcow_config = NULL;
+	char *mirage_qcow_stats_config = NULL;
 	struct mirage_block_stat msbuf;
 #endif
 
@@ -597,6 +598,8 @@ blockif_open(const char *optstr, const char *ident)
 			use_mirage = 1;
 		else if (strncmp(cp, "qcow-config=", 12) == 0)
 			mirage_qcow_config = cp + 12;
+		else if (strncmp(cp, "qcow-stats-config=", 18) == 0)
+			mirage_qcow_stats_config = cp + 18;
 #endif
 		else if (sscanf(cp, "sectorsize=%d/%d", &ssopt, &pssopt) == 2)
 			;
@@ -622,7 +625,7 @@ blockif_open(const char *optstr, const char *ident)
 	if (use_mirage) {
 #ifdef HAVE_OCAML_QCOW
 		mirage_block_register_thread();
-		mbh = mirage_block_open(nopt, mirage_qcow_config);
+		mbh = mirage_block_open(nopt, mirage_qcow_config, mirage_qcow_stats_config);
 		if (mbh < 0) {
 			perror("Could not open mirage-block device");
 			goto err;

vendor/github.com/moby/hyperkit/src/lib/mirage_block_c.c

@@ -46,19 +46,26 @@ if (fn == NULL) { \
 	acquiring the runtime lock. */
 
 static void
-ocaml_mirage_block_open(const char *config, const char *options, int *out, int *err) {
+ocaml_mirage_block_open(const char *device, const char *qcow_config, const char *stats_config, int *out, int *err) {
 	CAMLparam0();
-	CAMLlocal4(ocaml_config, ocaml_options_opt, ocaml_string, handle);
-	ocaml_config = caml_copy_string(config);
-	if (options == NULL) {
-		ocaml_options_opt = Val_int(0); /* None */
+	CAMLlocal5(ocaml_device, ocaml_qcow_config, ocaml_stats_config, ocaml_string, handle);
+	ocaml_device = caml_copy_string(device);
+	if (qcow_config == NULL) {
+		ocaml_qcow_config = Val_int(0); /* None */
 	} else {
-		ocaml_string = caml_copy_string(options);
-		ocaml_options_opt = caml_alloc(1, 0); /* Some */
-		Store_field (ocaml_options_opt, 0, ocaml_string);
+		ocaml_string = caml_copy_string(qcow_config);
+		ocaml_qcow_config = caml_alloc(1, 0); /* Some */
+		Store_field (ocaml_qcow_config, 0, ocaml_string);
+	}
+	if (stats_config == NULL) {
+		ocaml_stats_config = Val_int(0); /* None */
+	} else {
+		ocaml_string = caml_copy_string(stats_config);
+		ocaml_stats_config = caml_alloc(1, 0); /* Some */
+		Store_field (ocaml_stats_config, 0, ocaml_string);
 	}
 	OCAML_NAMED_FUNCTION("mirage_block_open")
-	handle = caml_callback2_exn(*fn, ocaml_config, ocaml_options_opt);
+	handle = caml_callback3_exn(*fn, ocaml_device, ocaml_qcow_config, ocaml_stats_config);
 	if (Is_exception_result(handle)){
 		*err = 1;
 	} else {
@@ -69,11 +76,11 @@ ocaml_mirage_block_open(const char *config, const char *options, int *out, int *
 }
 
 mirage_block_handle
-mirage_block_open(const char *config, const char *options) {
+mirage_block_open(const char *device, const char *qcow_config, const char *stats_config) {
 	int result;
 	int err = 1;
 	caml_acquire_runtime_system();
-	ocaml_mirage_block_open(config, options, &result, &err);
+	ocaml_mirage_block_open(device, qcow_config, stats_config, &result, &err);
 	caml_release_runtime_system();
 	if (err){
 		errno = EINVAL;

vendor/github.com/moby/hyperkit/src/lib/mirage_block_c.h

@@ -31,10 +31,12 @@ mirage_block_unregister_thread(void);
 /* An opened mirage-block device */
 typedef int mirage_block_handle;
 
-/* Open a mirage block device with the given optional string configuration.
-   To use the default configuration, pass NULL for options. */
+/* Open a mirage block device with the given optional qcow and stats
+   configuration.
+   To use the default configuration, pass NULL for qcow_config.
+   To not expose stats, pass NULL for stats_config */
 extern mirage_block_handle
-mirage_block_open(const char *config, const char *options);
+mirage_block_open(const char *device, const char *qcow_config, const char *stats_config);
 
 struct mirage_block_stat {
 	int candelete;      /* 1 if the device supports TRIM/DELETE/DISCARD */