Merge pull request #1547 from rneugeba/demo-up

Update etcd demo

projects/demo/etcd/Dockerfile

@@ -1,6 +1,10 @@
 # A dockerfile to build an etcd container image from the upstream one
 # with a script as entry point
 FROM quay.io/coreos/etcd:v3.1.5
+
+# sfdisk and jq re required to mount the disk
+RUN apk add --no-cache sfdisk jq
+
 COPY ./etcd.sh /
 
 ENTRYPOINT ["/etcd.sh"]

projects/demo/etcd/README.md

@@ -4,6 +4,9 @@ An `etcd` cluster can be bootstrapped in different ways (see the [Documentation]
 
 The moby `etcd` package is build with [build-pkg.sh](./build-pkg.sh). It takes the official `etcd` container and adds a [script](./etcd.sh) to start `etcd`. [etcd.sh](./etcd.sh) first attempts to join a new cluster. If that fails it attempts to join an existing cluster. Note, the number and members of the cluster are somewhat hard coded in the script.
 
+Each node is also configured with a disk, which is mounted inside the
+`etcd` container. `etcd` uses it to keep some state to help with
+restarts.
 
 ## Preparation
 
@@ -38,6 +41,9 @@ the nodes and then:
 docker run --rm -t quay.io/coreos/etcd:v3.1.5 etcdctl --endpoints http://192.168.65.200:2379 member list
 ```
 
+You can perform rolling updates, by for example, switching the kernel version in `etcd.yml`, build a new moby, e.g., `moby build -name etcd-4.10 etcd`, update `infrakit.json`, and then commit the new configuration to InfraKit: `infrakit group commit infrakit.json`.
+
+
 ## Infrakit GCP setup
 
 Note: This setup is somewhat specific to our GCP setup (IP addresses

projects/demo/etcd/etcd.sh

@@ -4,6 +4,39 @@
 set -x
 set -v
 
+MOUNTPOINT=/var/etcd
+
+mount_drive()
+{
+	mkdir -p "$MOUNTPOINT"
+
+	# TODO fix for multiple disks, cdroms etc
+	DEVS="$(find /dev -maxdepth 1 -type b ! -name 'loop*' ! -name 'nbd*' | grep -v '[0-9]$' | sed 's@.*/dev/@@' | sort)"
+
+	for DEV in $DEVS
+	do
+		DRIVE="/dev/${DEV}"
+
+		# see if it has a partition table
+		if sfdisk -d "${DRIVE}" >/dev/null 2>/dev/null
+		then
+			# 83 is Linux partition identifier
+			DATA=$(sfdisk -J "$DRIVE" | jq -e -r '.partitiontable.partitions | map(select(.type=="83")) | .[0].node')
+			if [ $? -eq 0 ]
+			then
+				mount "$DATA" "$MOUNTPOINT" && return
+			fi
+		fi
+	done
+
+	echo "WARNING: Failed to mount a persistent volume (is there one?)"
+
+	# not sure if we want to fatally bail here, in some debug situations it is ok
+	# exit 1
+}
+
+mount_drive
+
 # Wait till we have an IP address
 IP=""
 while [ -z "$IP" ]; do
@@ -17,13 +50,20 @@ PREFIX=$(echo ${IP} | cut -d . -f 1,2,3)
 NAME=infra${NUM}
 
 # This should come from Metadata
-INIT_CLUSTER=infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380,infra203=http://${PREFIX}.203:2380,infra204=http://${PREFIX}.204:2380
+INIT_CLUSTER="infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380,infra203=http://${PREFIX}.203:2380,infra204=http://${PREFIX}.204:2380"
 
-# Try to start in *new* cluster mode
+# We currently have no easy way to determine if we join a cluster for
+# the first time or if we got restarted and need to join an existing
+# cluster. So we first try joining a *new* cluster. This fails if we
+# had already joined it previously. As a fallback we then try to join
+# an existing cluster.
+
+# Try to join an new cluster
 /usr/local/bin/etcd \
     --name ${NAME} \
     --debug \
     --log-package-levels etcdmain=DEBUG,etcdserver=DEBUG \
+    --data-dir $MOUNTPOINT \
     --initial-advertise-peer-urls http://${IP}:2380 \
     --listen-peer-urls http://${IP}:2380 \
     --listen-client-urls http://${IP}:2379,http://127.0.0.1:2379 \
@@ -34,11 +74,12 @@ INIT_CLUSTER=infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:23
 
 [ $? -eq 0 ] && exit 0
 
-# Joining the new cluster failed. Let's try joining an *existing* cluster
+# Try to join an existing cluster
 /usr/local/bin/etcd \
     --name ${NAME} \
     --debug \
     --log-package-levels etcdmain=DEBUG,etcdserver=DEBUG \
+    --data-dir $MOUNTPOINT \
     --initial-advertise-peer-urls http://${IP}:2380 \
     --listen-peer-urls http://${IP}:2380 \
     --listen-client-urls http://${IP}:2379,http://127.0.0.1:2379 \

projects/demo/etcd/etcd.yml

@@ -2,7 +2,9 @@ kernel:
   image: "mobylinux/kernel:4.9.x"
   cmdline: "console=ttyS0 console=tty0 page_poison=1"
 init:
-  - "mobylinux/init:a27e32a8d6c8865d691fbfb4d0bbb93846cf7802"
+  - mobylinux/init:925c88f42d92d57cd36b656db1f8757b152163a7
+  - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9
+  - mobylinux/containerd:68bb523deea09da293d675cbf88474eced540b8c
 onboot:
   - name: sysctl
     image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c"
@@ -12,6 +14,13 @@ onboot:
     capabilities:
      - CAP_SYS_ADMIN
     readonly: true
+  - name: format
+    image: "mobylinux/format:53748000acf515549d398e6ae68545c26c0f3a2e"
+    binds:
+     - /dev:/dev
+    capabilities:
+     - CAP_SYS_ADMIN
+     - CAP_MKNOD
 services:
   - name: rngd
     image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92"
@@ -33,12 +42,15 @@ services:
   - name: etcd
     image: "mobylinux/etcd"
     capabilities:
-     - CAP_NET_BIND_SERVICE
      - CAP_CHOWN
      - CAP_SETUID
      - CAP_SETGID
      - CAP_DAC_OVERRIDE
+     - CAP_SYS_ADMIN
+     - CAP_MKNOD
     net: host
+    binds:
+     - /dev:/dev
 outputs:
   - format: kernel+initrd
 #  - format: gcp

projects/demo/etcd/infrakit.json

@@ -14,7 +14,7 @@
             "Plugin": "instance-hyperkit",
             "Properties": {
                 "Moby": "etcd",
-                "Disk" : 0,
+                "Disk" : 1024,
                 "CPUs" : 1,
                 "Memory" : 1024
             }

projects/demo/etcd/start-infrakit.sh

@@ -6,6 +6,7 @@ INFRAKIT_HOME=~/.infrakit
 IK_PLUGINS=$INFRAKIT_HOME/plugins
 
 rm -rf $INFRAKIT_HOME
+mkdir -p $INFRAKIT_HOME/cli
 
 infrakit-flavor-vanilla &
 infrakit-instance-hyperkit &

projects/demo/intro/intro.yml

@@ -2,7 +2,9 @@ kernel:
   image: "mobylinux/kernel:4.9.x"
   cmdline: "console=ttyS0 console=tty0 page_poison=1"
 init:
-  - "mobylinux/init:a27e32a8d6c8865d691fbfb4d0bbb93846cf7802"
+  - mobylinux/init:925c88f42d92d57cd36b656db1f8757b152163a7
+  - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9
+  - mobylinux/containerd:68bb523deea09da293d675cbf88474eced540b8c
 onboot:
   - name: sysctl
     image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c"

src/cmd/infrakit-instance-hyperkit/instance.go

@@ -21,6 +21,7 @@ func NewHyperKitPlugin(vmDir, hyperkit, vpnkitSock string) instance.Plugin {
 	return &hyperkitPlugin{VMDir: vmDir,
 		HyperKit:   hyperkit,
 		VPNKitSock: vpnkitSock,
+		DiskDir:    path.Join(vmDir, "disks"),
 	}
 }
 
@@ -33,6 +34,9 @@ type hyperkitPlugin struct {
 
 	// VPNKitSock is the path to the VPNKit Unix domain socket.
 	VPNKitSock string
+
+	// DiskDir is the path to persistent (across reboots) disk images
+	DiskDir string
 }
 
 // Validate performs local validation on a provision request.
@@ -60,8 +64,9 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 	if properties["Memory"] == nil {
 		properties["Memory"] = 512
 	}
-	if properties["Disk"] == nil {
+	diskSize := 0
-		properties["Disk"] = 256
+	if properties["Disk"] != nil {
+		diskSize = int(properties["Disk"].(float64))
 	}
 
 	instanceDir, err := ioutil.TempDir(p.VMDir, "infrakit-")
@@ -71,12 +76,15 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 	id := instance.ID(path.Base(instanceDir))
 	log.Infof("[%s] New instance", id)
 
-	// The LogicalID may be a IP address. If so, translate it into a
-	// magic UUID which cause VPNKit to assign a fixed IP address
 	logicalID := string(id)
 	uuidStr := ""
+
+	diskImage := ""
 	if spec.LogicalID != nil {
 		logicalID = string(*spec.LogicalID)
+		// The LogicalID may be a IP address. If so, translate
+		// it into a magic UUID which cause VPNKit to assign a
+		// fixed IP address
 		if ip := net.ParseIP(logicalID); len(ip) > 0 {
 			uuid := make([]byte, 16)
 			uuid[12] = ip.To4()[0]
@@ -85,12 +93,23 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 			uuid[15] = ip.To4()[3]
 			uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:])
 		}
+		// If a LogicalID is supplied and the Disk size is
+		// non-zero, we place the disk in a special directory
+		// so it persists across reboots.
+		if diskSize != 0 {
+			diskImage = path.Join(p.DiskDir, logicalID+".img")
+			// Make sure the directory exists
+			err = os.MkdirAll(p.DiskDir, 0755)
+			if err != nil {
+				return nil, err
+			}
+		}
 	}
 	log.Infof("[%s] LogicalID: %s", id, logicalID)
 	log.Debugf("[%s] UUID: %s", id, uuidStr)
 
 	// Start a HyperKit instance
-	h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, "")
+	h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, diskImage)
 	if err != nil {
 		return nil, err
 	}
@@ -98,12 +117,12 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) {
 	h.Initrd = properties["Moby"].(string) + "-initrd.img"
 	h.CPUs = int(properties["CPUs"].(float64))
 	h.Memory = int(properties["Memory"].(float64))
-	h.DiskSize = int(properties["Disk"].(float64))
+	h.DiskSize = diskSize
 	h.UUID = uuidStr
 	h.UserData = spec.Init
 	h.Console = hyperkit.ConsoleFile
 	log.Infof("[%s] Booting: %s/%s", id, h.Kernel, h.Initrd)
-	log.Infof("[%s] %d CPUs, %dMB Memory, %dMB Disk", id, h.CPUs, h.Memory, h.DiskSize)
+	log.Infof("[%s] %d CPUs, %dMB Memory, %dMB Disk (%s)", id, h.CPUs, h.Memory, h.DiskSize, h.DiskImage)
 
 	err = h.Start("console=ttyS0")
 	if err != nil {