Merge pull request #1529 from rneugeba/hk-infra

Update Hyperkit bindings and demo

projects/demo/README.md

@@ -5,6 +5,8 @@ This directory contains files used in moby demos.
 Most of the scripts/files assume you are on a Mac.
 
 - Recent Docker for Mac installed (edge, nightly, master channel)
+- Currently, you need a *custom* version of VPNKit installed in Docker
+  for Mac. Contact @magnus or @rneugeba.
 - For the GCP portion: `brew install google-cloud-sdk`
 - For `etcd`: `brew install etcd`
 - Infrakit: Clone [infrakit](https://github.com/docker/infrakit) and

projects/demo/etcd/README.md

@@ -1,57 +1,33 @@
-This directory contains scripts/files to bootstrap a `etcd` cluster.
+This directory contains scripts/files to bootstrap a `etcd` cluster both on the local machine as well as on Google Cloud.
 
-In the local, hyperkit based, setup, we use a `etcd` running in a
+An `etcd` cluster can be bootstrapped in different ways (see the [Documentatiob](https://coreos.com/etcd/docs/latest/op-guide/clustering.html) for more details. For the demo we use configuration via static IP addresses. With Infrakit these are managed by assigning `LogicalID`s to cluster members. The `LogicalID` is interpreted as a IP address.
-Docker for Mac container to bootstrap the cluster. For a cloud based demo, we'd use `https://discovery.etcd.io`. The host/DfM side is setup with [dfm-setup.sh](./dfm-setup.sh).
 
-The moby `etcd` package is build with [build-pkg.sh](./build-pkg.sh). It take the official `etcd` container and adds a [script](./etcd.sh) to start `etcd`.
+The moby `etcd` package is build with [build-pkg.sh](./build-pkg.sh). It takes the official `etcd` container and adds a [script](./etcd.sh) to start `etcd`. [etcd.sh](./etcd.sh) first attempts to join a new cluster. If that fails it attempts to join an existing cluster. Note, the number and members of the cluster are somewhat hard coded in the script.
 
 
-## Simple single node cluster (OUTDATED)
+## Preparation
 
-- Edit `./dfm-setup.sh` and set `NUMPEERS` to `1`
+- Build the `etcd` image and then moby image:
-- Start the etcd bootstrap container in on window:
-```
-./dfm-setup.sh
-```
-
-- In another window build/run the moby image:
 ```
 ./build-pkg.sh
 moby build etcd
-moby run etcd
 ```
 
 ## InfraKit cluster setup (OUTDATED)
 
-This should create a HyperKit based, InfraKit managed `etcd` cluster with 5 `etcd` instances.
+This should create a HyperKit based, InfraKit managed `etcd` cluster with 3 `etcd` instances.
 
-#### Infrakit setup
+Start InfraKit:
-You need the [infrakit](https://github.com/docker/infrakit) binaries for this. I normally compile from source using `make build-in-container`. The below was tried with commit `cb420e3e50ea60afe58538b1d3cab1cb14059433`.
-
-- Make sure you start from scratch
-```
-rm -rf ~/.infrakit
-```
-- Start the infrakit plugins, each in it's own window from the root of the infrakit source tree:
-```
-infrakit-group-default
-```
-```
-infrakit-flavor-vanilla
-```
-- Start the hyperkit instance plugin from this directory:
 ```
+infrakit-flavor-vanilla &
+infrakit-group-default &
 ../../../bin/infrakit-instance-hyperkit
 ```
 
-#### etcd setup
+Note: The HyperKit InfraKit plugin must be started from the directory
+where the `etcd` mobylinux image is located.
 
-- Start the bootstrap `etcd`:
+Now, commit the new config:
-```
-./dfm-setup.sh
-```
-
-- Commit the infrakit config:
 ```
 infrakit group commit infrakit.json
 ```
@@ -59,7 +35,7 @@ infrakit group commit infrakit.json
 To check if everything is fine, note down the IP address from one of
 the nodes and then:
 ```
-docker run --rm -t quay.io/coreos/etcd:v3.1.5 etcdctl --endpoints http://192.168.65.24:2379 member list
+docker run --rm -t quay.io/coreos/etcd:v3.1.5 etcdctl --endpoints http://192.168.65.200:2379 member list
 ```
 
 ## Infrakit GCP setup

projects/demo/etcd/etcd.sh

@@ -29,3 +29,18 @@ NAME=infra${NUM}
     --initial-cluster-token etcd-cluster-1 \
     --initial-cluster infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380 \
     --initial-cluster-state new
+
+echo "Returned $?"
+
+# If we get here, joining a new cluster failed. Let's try joining an
+# existing cluster
+/usr/local/bin/etcd \
+    --name ${NAME} \
+    --debug \
+    --log-package-levels etcdmain=DEBUG,etcdserver=DEBUG \
+    --initial-advertise-peer-urls http://${IP}:2380 \
+    --listen-peer-urls http://${IP}:2380 \
+    --listen-client-urls http://${IP}:2379,http://127.0.0.1:2379 \
+    --advertise-client-urls http://${IP}:2379 \
+    --initial-cluster infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380 \
+    --initial-cluster-state existing

projects/demo/etcd/infrakit.json

@@ -14,7 +14,7 @@
                 "Moby": "etcd",
                 "Disk" : 0,
                 "CPUs" : 1,
-                "Memory" : 512
+                "Memory" : 1024
             }
         },
         "Flavor": {

src/cmd/infrakit-instance-hyperkit/instance.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"path"
 
@@ -70,11 +71,23 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 	id := instance.ID(path.Base(instanceDir))
 	log.Infof("[%s] New instance", id)
 
+	// The LogicalID may be a IP address. If so, translate it into a
+	// magic UUID which cause VPNKit to assign a fixed IP address
 	logicalID := string(id)
+	uuidStr := ""
 	if spec.LogicalID != nil {
 		logicalID = string(*spec.LogicalID)
+		if ip := net.ParseIP(logicalID); len(ip) > 0 {
+			uuid := make([]byte, 16)
+			uuid[12] = ip.To4()[0]
+			uuid[13] = ip.To4()[1]
+			uuid[14] = ip.To4()[2]
+			uuid[15] = ip.To4()[3]
+			uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
+		}
 	}
 	log.Infof("[%s] LogicalID: %s", id, logicalID)
+	log.Debugf("[%s] UUID: %s", id, uuidStr)
 
 	// Start a HyperKit instance
 	h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, "")
@@ -86,6 +99,7 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 	h.CPUs = int(properties["CPUs"].(float64))
 	h.Memory = int(properties["Memory"].(float64))
 	h.DiskSize = int(properties["Disk"].(float64))
+	h.UUID = uuidStr
 	h.UserData = spec.Init
 	h.Console = hyperkit.ConsoleFile
 	log.Infof("[%s] Booting: %s/%s", id, h.Kernel, h.Initrd)

src/cmd/moby/run_hyperkit.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 
 	log "github.com/Sirupsen/logrus"
@@ -27,6 +28,7 @@ func runHyperKit(args []string) {
 	diskSz := hyperkitCmd.Int("disk-size", 0, "Size of Disk in MB")
 	disk := hyperkitCmd.String("disk", "", "Path to disk image to used")
 	data := hyperkitCmd.String("data", "", "Metadata to pass to VM (either a path to a file or a string)")
+	ipStr := hyperkitCmd.String("ip", "", "IP address for the VM")
 
 	hyperkitCmd.Parse(args)
 	remArgs := hyperkitCmd.Args()
@@ -60,6 +62,20 @@ func runHyperKit(args []string) {
 		outfh.Close()
 	}
 
+	uuidStr := ""
+	if *ipStr != "" {
+		// If an IP address was requested construct a "special" UUID
+		// for the VM.
+		if ip := net.ParseIP(*ipStr); len(ip) > 0 {
+			uuid := make([]byte, 16)
+			uuid[12] = ip.To4()[0]
+			uuid[13] = ip.To4()[1]
+			uuid[14] = ip.To4()[2]
+			uuid[15] = ip.To4()[3]
+			uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
+		}
+	}
+
 	// Run
 	cmdline, err := ioutil.ReadFile(prefix + "-cmdline")
 	if err != nil {
@@ -81,6 +97,7 @@ func runHyperKit(args []string) {
 	h.CPUs = *cpus
 	h.Memory = *mem
 	h.DiskSize = *diskSz
+	h.UUID = uuidStr
 
 	err = h.Run(string(cmdline))
 	if err != nil {

vendor.conf

@@ -9,7 +9,7 @@ github.com/docker/distribution 07f32ac1831ed0fc71960b7da5d6bb83cb6881b5
 github.com/docker/engine-api cf82c64276ebc2501e72b241f9fdc1e21e421743
 github.com/docker/go-connections e15c02316c12de00874640cd76311849de2aeed5
 github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3
-github.com/docker/hyperkit/go 57e91c5bb6655514aa71d00dd1949db891903d34
+github.com/docker/hyperkit/go 89b0a4a88ec340c756a6bd14af57e26f69d29d09
 github.com/docker/infrakit cb420e3e50ea60afe58538b1d3cab1cb14059433
 github.com/ghodss/yaml 0ca9ea5df5451ffdf184b4428c902747c2c11cd7
 github.com/golang/protobuf/proto c9c7427a2a70d2eb3bafa0ab2dc163e45f143317

vendor/github.com/docker/hyperkit/go/hyperkit.go

@@ -69,6 +69,8 @@ type HyperKit struct {
 	StateDir string `json:"state_dir"`
 	// VPNKitSock is the location of the VPNKit socket used for networking.
 	VPNKitSock string `json:"vpnkit_sock"`
+	// UUID is a string containing a UUID for the VM. It can be used in conjunction with VPNKit to get consistent IP address.
+	UUID string `json:"uuid"`
 	// DiskImage is the path to the disk image to use
 	DiskImage string `json:"disk"`
 	// ISOImage is the (optional) path to a ISO image to attach
@@ -144,12 +146,12 @@ func New(hyperkit, statedir, vpnkitsock, diskimage string) (*HyperKit, error) {
 func FromState(statedir string) (*HyperKit, error) {
 	b, err := ioutil.ReadFile(filepath.Join(statedir, jsonFile))
 	if err != nil {
-		return nil, fmt.Errorf("Can't read json file: ", err)
+		return nil, fmt.Errorf("Can't read json file: %s", err)
 	}
 	h := &HyperKit{}
 	err = json.Unmarshal(b, h)
 	if err != nil {
-		return nil, fmt.Errorf("Can't parse json file: ", err)
+		return nil, fmt.Errorf("Can't parse json file: %s", err)
 	}
 
 	// Make sure the pid written by hyperkit is the same as in the json
@@ -353,7 +355,11 @@ func (h *HyperKit) buildArgs(cmdline string) {
 
 	a = append(a, "-s", "0:0,hostbridge")
 	if h.VPNKitSock != "" {
-		a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s", h.VPNKitSock))
+		if h.UUID == "" {
+			a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s", h.VPNKitSock))
+		} else {
+			a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s,uuid=%s", h.VPNKitSock, h.UUID))
+		}
 	}
 	if h.DiskImage != "" {
 		a = append(a, "-s", fmt.Sprintf("2:0,virtio-blk,%s", h.DiskImage))
@@ -502,7 +508,7 @@ func checkHyperKit(hyperkit string) (string, error) {
 	if hyperkit != "" {
 		p, err := exec.LookPath(hyperkit)
 		if err != nil {
-			return "", fmt.Errorf("Could not find hyperkit executable: ", hyperkit)
+			return "", fmt.Errorf("Could not find hyperkit executable %s: %s", hyperkit, err)
 		}
 		return p, nil
 	}