From 569a5a4ba86400aaa7398f6c6ca67ee8cd4b3e46 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Thu, 6 Apr 2017 21:45:14 +0100 Subject: [PATCH 1/6] vendor: Update the HyperKit go bindings Signed-off-by: Rolf Neugebauer --- vendor.conf | 2 +- vendor/github.com/docker/hyperkit/go/hyperkit.go | 14 ++++++++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/vendor.conf b/vendor.conf index 325d8235b..3bf35ba7a 100644 --- a/vendor.conf +++ b/vendor.conf @@ -9,7 +9,7 @@ github.com/docker/distribution 07f32ac1831ed0fc71960b7da5d6bb83cb6881b5 github.com/docker/engine-api cf82c64276ebc2501e72b241f9fdc1e21e421743 github.com/docker/go-connections e15c02316c12de00874640cd76311849de2aeed5 github.com/docker/go-units 651fc226e7441360384da338d0fd37f2440ffbe3 -github.com/docker/hyperkit/go 57e91c5bb6655514aa71d00dd1949db891903d34 +github.com/docker/hyperkit/go 89b0a4a88ec340c756a6bd14af57e26f69d29d09 github.com/docker/infrakit cb420e3e50ea60afe58538b1d3cab1cb14059433 github.com/ghodss/yaml 0ca9ea5df5451ffdf184b4428c902747c2c11cd7 github.com/golang/protobuf/proto c9c7427a2a70d2eb3bafa0ab2dc163e45f143317 diff --git a/vendor/github.com/docker/hyperkit/go/hyperkit.go b/vendor/github.com/docker/hyperkit/go/hyperkit.go index 54cf6a32e..f79ff94b1 100644 --- a/vendor/github.com/docker/hyperkit/go/hyperkit.go +++ b/vendor/github.com/docker/hyperkit/go/hyperkit.go @@ -69,6 +69,8 @@ type HyperKit struct { StateDir string `json:"state_dir"` // VPNKitSock is the location of the VPNKit socket used for networking. VPNKitSock string `json:"vpnkit_sock"` + // UUID is a string containing a UUID for the VM. It can be used in conjunction with VPNKit to get consistent IP address. + UUID string `json:"uuid"` // DiskImage is the path to the disk image to use DiskImage string `json:"disk"` // ISOImage is the (optional) path to a ISO image to attach @@ -144,12 +146,12 @@ func New(hyperkit, statedir, vpnkitsock, diskimage string) (*HyperKit, error) { func FromState(statedir string) (*HyperKit, error) { b, err := ioutil.ReadFile(filepath.Join(statedir, jsonFile)) if err != nil { - return nil, fmt.Errorf("Can't read json file: ", err) + return nil, fmt.Errorf("Can't read json file: %s", err) } h := &HyperKit{} err = json.Unmarshal(b, h) if err != nil { - return nil, fmt.Errorf("Can't parse json file: ", err) + return nil, fmt.Errorf("Can't parse json file: %s", err) } // Make sure the pid written by hyperkit is the same as in the json @@ -353,7 +355,11 @@ func (h *HyperKit) buildArgs(cmdline string) { a = append(a, "-s", "0:0,hostbridge") if h.VPNKitSock != "" { - a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s", h.VPNKitSock)) + if h.UUID == "" { + a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s", h.VPNKitSock)) + } else { + a = append(a, "-s", fmt.Sprintf("1:0,virtio-vpnkit,path=%s,uuid=%s", h.VPNKitSock, h.UUID)) + } } if h.DiskImage != "" { a = append(a, "-s", fmt.Sprintf("2:0,virtio-blk,%s", h.DiskImage)) @@ -502,7 +508,7 @@ func checkHyperKit(hyperkit string) (string, error) { if hyperkit != "" { p, err := exec.LookPath(hyperkit) if err != nil { - return "", fmt.Errorf("Could not find hyperkit executable: ", hyperkit) + return "", fmt.Errorf("Could not find hyperkit executable %s: %s", hyperkit, err) } return p, nil } From 971db3c0c3d862638e29633038c560c256e83baa Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Thu, 6 Apr 2017 23:16:32 +0100 Subject: [PATCH 2/6] infrakit: Add LogicalID -> IP -> UUID mapping If the LogicalID passed in is a IP address, turn this into a UUID and pass it to HyperKit. This will cause VPNKit to assign the IP address the VM. Note: This currently requires a custom version of VPNKit Signed-off-by: Rolf Neugebauer --- src/cmd/infrakit-instance-hyperkit/instance.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/cmd/infrakit-instance-hyperkit/instance.go b/src/cmd/infrakit-instance-hyperkit/instance.go index b8c28378c..6b1d4d1f7 100644 --- a/src/cmd/infrakit-instance-hyperkit/instance.go +++ b/src/cmd/infrakit-instance-hyperkit/instance.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "io/ioutil" + "net" "os" "path" @@ -70,11 +71,23 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) { id := instance.ID(path.Base(instanceDir)) log.Infof("[%s] New instance", id) + // The LogicalID may be a IP address. If so, translate it into a + // magic UUID which cause VPNKit to assign a fixed IP address logicalID := string(id) + uuidStr := "" if spec.LogicalID != nil { logicalID = string(*spec.LogicalID) + if ip := net.ParseIP(logicalID); len(ip) > 0 { + uuid := make([]byte, 16) + uuid[12] = ip.To4()[0] + uuid[13] = ip.To4()[1] + uuid[14] = ip.To4()[2] + uuid[15] = ip.To4()[3] + uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]) + } } log.Infof("[%s] LogicalID: %s", id, logicalID) + log.Debugf("[%s] UUID: %s", id, uuidStr) // Start a HyperKit instance h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, "") @@ -86,6 +99,7 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) { h.CPUs = int(properties["CPUs"].(float64)) h.Memory = int(properties["Memory"].(float64)) h.DiskSize = int(properties["Disk"].(float64)) + h.UUID = uuidStr h.UserData = spec.Init h.Console = hyperkit.ConsoleFile log.Infof("[%s] Booting: %s/%s", id, h.Kernel, h.Initrd) From cc16ceb5782c2863e1a954897dfc6e74de61273a Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Fri, 7 Apr 2017 09:25:59 +0100 Subject: [PATCH 3/6] cli: Add option to the IP address for HyperKit VM Note: This currently requires a custom version of VPNKit. Signed-off-by: Rolf Neugebauer --- src/cmd/moby/run_hyperkit.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/cmd/moby/run_hyperkit.go b/src/cmd/moby/run_hyperkit.go index ab9ded647..12faea39d 100644 --- a/src/cmd/moby/run_hyperkit.go +++ b/src/cmd/moby/run_hyperkit.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "io/ioutil" + "net" "os" log "github.com/Sirupsen/logrus" @@ -27,6 +28,7 @@ func runHyperKit(args []string) { diskSz := hyperkitCmd.Int("disk-size", 0, "Size of Disk in MB") disk := hyperkitCmd.String("disk", "", "Path to disk image to used") data := hyperkitCmd.String("data", "", "Metadata to pass to VM (either a path to a file or a string)") + ipStr := hyperkitCmd.String("ip", "", "IP address for the VM") hyperkitCmd.Parse(args) remArgs := hyperkitCmd.Args() @@ -60,6 +62,20 @@ func runHyperKit(args []string) { outfh.Close() } + uuidStr := "" + if *ipStr != "" { + // If an IP address was requested construct a "special" UUID + // for the VM. + if ip := net.ParseIP(*ipStr); len(ip) > 0 { + uuid := make([]byte, 16) + uuid[12] = ip.To4()[0] + uuid[13] = ip.To4()[1] + uuid[14] = ip.To4()[2] + uuid[15] = ip.To4()[3] + uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]) + } + } + // Run cmdline, err := ioutil.ReadFile(prefix + "-cmdline") if err != nil { @@ -81,6 +97,7 @@ func runHyperKit(args []string) { h.CPUs = *cpus h.Memory = *mem h.DiskSize = *diskSz + h.UUID = uuidStr err = h.Run(string(cmdline)) if err != nil { From 17e4aa3e589e31e9ec0de64871ec4c3bdabe278c Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Thu, 6 Apr 2017 23:19:28 +0100 Subject: [PATCH 4/6] demo: Increase memory of etcd VMs to 1G 512MB is too small to install etcd via apk in the RAM disk. Signed-off-by: Rolf Neugebauer --- projects/demo/etcd/infrakit.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/demo/etcd/infrakit.json b/projects/demo/etcd/infrakit.json index c4844dd7e..d0d709e91 100644 --- a/projects/demo/etcd/infrakit.json +++ b/projects/demo/etcd/infrakit.json @@ -14,7 +14,7 @@ "Moby": "etcd", "Disk" : 0, "CPUs" : 1, - "Memory" : 512 + "Memory" : 1024 } }, "Flavor": { From 01e39a16acf4ac523575058f1c0ec4f44b199b97 Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Fri, 7 Apr 2017 01:19:36 +0100 Subject: [PATCH 5/6] demo: Tweak etcd start script Try joining a new cluster initially. If that fails try to join an existing cluster. Signed-off-by: Rolf Neugebauer --- projects/demo/etcd/etcd.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/projects/demo/etcd/etcd.sh b/projects/demo/etcd/etcd.sh index a2d6ad05e..8003145d9 100755 --- a/projects/demo/etcd/etcd.sh +++ b/projects/demo/etcd/etcd.sh @@ -29,3 +29,18 @@ NAME=infra${NUM} --initial-cluster-token etcd-cluster-1 \ --initial-cluster infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380 \ --initial-cluster-state new + +echo "Returned $?" + +# If we get here, joining a new cluster failed. Let's try joining an +# existing cluster +/usr/local/bin/etcd \ + --name ${NAME} \ + --debug \ + --log-package-levels etcdmain=DEBUG,etcdserver=DEBUG \ + --initial-advertise-peer-urls http://${IP}:2380 \ + --listen-peer-urls http://${IP}:2380 \ + --listen-client-urls http://${IP}:2379,http://127.0.0.1:2379 \ + --advertise-client-urls http://${IP}:2379 \ + --initial-cluster infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380 \ + --initial-cluster-state existing From 93a9a827b65f1a423d72710a32c99b20678739fd Mon Sep 17 00:00:00 2001 From: Rolf Neugebauer Date: Fri, 7 Apr 2017 09:40:58 +0100 Subject: [PATCH 6/6] demo: Update documentation Signed-off-by: Rolf Neugebauer --- projects/demo/README.md | 2 ++ projects/demo/etcd/README.md | 50 ++++++++++-------------------------- 2 files changed, 15 insertions(+), 37 deletions(-) diff --git a/projects/demo/README.md b/projects/demo/README.md index a20a4fed0..18f1bcaca 100644 --- a/projects/demo/README.md +++ b/projects/demo/README.md @@ -5,6 +5,8 @@ This directory contains files used in moby demos. Most of the scripts/files assume you are on a Mac. - Recent Docker for Mac installed (edge, nightly, master channel) +- Currently, you need a *custom* version of VPNKit installed in Docker + for Mac. Contact @magnus or @rneugeba. - For the GCP portion: `brew install google-cloud-sdk` - For `etcd`: `brew install etcd` - Infrakit: Clone [infrakit](https://github.com/docker/infrakit) and diff --git a/projects/demo/etcd/README.md b/projects/demo/etcd/README.md index 0f969013c..8b93ceab2 100644 --- a/projects/demo/etcd/README.md +++ b/projects/demo/etcd/README.md @@ -1,57 +1,33 @@ -This directory contains scripts/files to bootstrap a `etcd` cluster. +This directory contains scripts/files to bootstrap a `etcd` cluster both on the local machine as well as on Google Cloud. -In the local, hyperkit based, setup, we use a `etcd` running in a -Docker for Mac container to bootstrap the cluster. For a cloud based demo, we'd use `https://discovery.etcd.io`. The host/DfM side is setup with [dfm-setup.sh](./dfm-setup.sh). +An `etcd` cluster can be bootstrapped in different ways (see the [Documentatiob](https://coreos.com/etcd/docs/latest/op-guide/clustering.html) for more details. For the demo we use configuration via static IP addresses. With Infrakit these are managed by assigning `LogicalID`s to cluster members. The `LogicalID` is interpreted as a IP address. -The moby `etcd` package is build with [build-pkg.sh](./build-pkg.sh). It take the official `etcd` container and adds a [script](./etcd.sh) to start `etcd`. +The moby `etcd` package is build with [build-pkg.sh](./build-pkg.sh). It takes the official `etcd` container and adds a [script](./etcd.sh) to start `etcd`. [etcd.sh](./etcd.sh) first attempts to join a new cluster. If that fails it attempts to join an existing cluster. Note, the number and members of the cluster are somewhat hard coded in the script. -## Simple single node cluster (OUTDATED) +## Preparation -- Edit `./dfm-setup.sh` and set `NUMPEERS` to `1` -- Start the etcd bootstrap container in on window: -``` -./dfm-setup.sh -``` - -- In another window build/run the moby image: +- Build the `etcd` image and then moby image: ``` ./build-pkg.sh moby build etcd -moby run etcd ``` ## InfraKit cluster setup (OUTDATED) -This should create a HyperKit based, InfraKit managed `etcd` cluster with 5 `etcd` instances. +This should create a HyperKit based, InfraKit managed `etcd` cluster with 3 `etcd` instances. -#### Infrakit setup -You need the [infrakit](https://github.com/docker/infrakit) binaries for this. I normally compile from source using `make build-in-container`. The below was tried with commit `cb420e3e50ea60afe58538b1d3cab1cb14059433`. - -- Make sure you start from scratch -``` -rm -rf ~/.infrakit -``` -- Start the infrakit plugins, each in it's own window from the root of the infrakit source tree: -``` -infrakit-group-default -``` -``` -infrakit-flavor-vanilla -``` -- Start the hyperkit instance plugin from this directory: +Start InfraKit: ``` +infrakit-flavor-vanilla & +infrakit-group-default & ../../../bin/infrakit-instance-hyperkit ``` -#### etcd setup +Note: The HyperKit InfraKit plugin must be started from the directory +where the `etcd` mobylinux image is located. -- Start the bootstrap `etcd`: -``` -./dfm-setup.sh -``` - -- Commit the infrakit config: +Now, commit the new config: ``` infrakit group commit infrakit.json ``` @@ -59,7 +35,7 @@ infrakit group commit infrakit.json To check if everything is fine, note down the IP address from one of the nodes and then: ``` -docker run --rm -t quay.io/coreos/etcd:v3.1.5 etcdctl --endpoints http://192.168.65.24:2379 member list +docker run --rm -t quay.io/coreos/etcd:v3.1.5 etcdctl --endpoints http://192.168.65.200:2379 member list ``` ## Infrakit GCP setup