diff --git a/projects/demo/etcd/Dockerfile b/projects/demo/etcd/Dockerfile index bee96b5dd..bc6fcca43 100644 --- a/projects/demo/etcd/Dockerfile +++ b/projects/demo/etcd/Dockerfile @@ -1,6 +1,10 @@ # A dockerfile to build an etcd container image from the upstream one # with a script as entry point FROM quay.io/coreos/etcd:v3.1.5 + +# sfdisk and jq re required to mount the disk +RUN apk add --no-cache sfdisk jq + COPY ./etcd.sh / ENTRYPOINT ["/etcd.sh"] diff --git a/projects/demo/etcd/README.md b/projects/demo/etcd/README.md index 6f6f013b9..c4b5ade77 100644 --- a/projects/demo/etcd/README.md +++ b/projects/demo/etcd/README.md @@ -4,6 +4,9 @@ An `etcd` cluster can be bootstrapped in different ways (see the [Documentation] The moby `etcd` package is build with [build-pkg.sh](./build-pkg.sh). It takes the official `etcd` container and adds a [script](./etcd.sh) to start `etcd`. [etcd.sh](./etcd.sh) first attempts to join a new cluster. If that fails it attempts to join an existing cluster. Note, the number and members of the cluster are somewhat hard coded in the script. +Each node is also configured with a disk, which is mounted inside the +`etcd` container. `etcd` uses it to keep some state to help with +restarts. ## Preparation @@ -38,6 +41,9 @@ the nodes and then: docker run --rm -t quay.io/coreos/etcd:v3.1.5 etcdctl --endpoints http://192.168.65.200:2379 member list ``` +You can perform rolling updates, by for example, switching the kernel version in `etcd.yml`, build a new moby, e.g., `moby build -name etcd-4.10 etcd`, update `infrakit.json`, and then commit the new configuration to InfraKit: `infrakit group commit infrakit.json`. + + ## Infrakit GCP setup Note: This setup is somewhat specific to our GCP setup (IP addresses diff --git a/projects/demo/etcd/etcd.sh b/projects/demo/etcd/etcd.sh index e93641c0c..40957ffd2 100755 --- a/projects/demo/etcd/etcd.sh +++ b/projects/demo/etcd/etcd.sh @@ -4,6 +4,39 @@ set -x set -v +MOUNTPOINT=/var/etcd + +mount_drive() +{ + mkdir -p "$MOUNTPOINT" + + # TODO fix for multiple disks, cdroms etc + DEVS="$(find /dev -maxdepth 1 -type b ! -name 'loop*' ! -name 'nbd*' | grep -v '[0-9]$' | sed 's@.*/dev/@@' | sort)" + + for DEV in $DEVS + do + DRIVE="/dev/${DEV}" + + # see if it has a partition table + if sfdisk -d "${DRIVE}" >/dev/null 2>/dev/null + then + # 83 is Linux partition identifier + DATA=$(sfdisk -J "$DRIVE" | jq -e -r '.partitiontable.partitions | map(select(.type=="83")) | .[0].node') + if [ $? -eq 0 ] + then + mount "$DATA" "$MOUNTPOINT" && return + fi + fi + done + + echo "WARNING: Failed to mount a persistent volume (is there one?)" + + # not sure if we want to fatally bail here, in some debug situations it is ok + # exit 1 +} + +mount_drive + # Wait till we have an IP address IP="" while [ -z "$IP" ]; do @@ -17,13 +50,20 @@ PREFIX=$(echo ${IP} | cut -d . -f 1,2,3) NAME=infra${NUM} # This should come from Metadata -INIT_CLUSTER=infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380,infra203=http://${PREFIX}.203:2380,infra204=http://${PREFIX}.204:2380 +INIT_CLUSTER="infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:2380,infra202=http://${PREFIX}.202:2380,infra203=http://${PREFIX}.203:2380,infra204=http://${PREFIX}.204:2380" -# Try to start in *new* cluster mode +# We currently have no easy way to determine if we join a cluster for +# the first time or if we got restarted and need to join an existing +# cluster. So we first try joining a *new* cluster. This fails if we +# had already joined it previously. As a fallback we then try to join +# an existing cluster. + +# Try to join an new cluster /usr/local/bin/etcd \ --name ${NAME} \ --debug \ --log-package-levels etcdmain=DEBUG,etcdserver=DEBUG \ + --data-dir $MOUNTPOINT \ --initial-advertise-peer-urls http://${IP}:2380 \ --listen-peer-urls http://${IP}:2380 \ --listen-client-urls http://${IP}:2379,http://127.0.0.1:2379 \ @@ -34,11 +74,12 @@ INIT_CLUSTER=infra200=http://${PREFIX}.200:2380,infra201=http://${PREFIX}.201:23 [ $? -eq 0 ] && exit 0 -# Joining the new cluster failed. Let's try joining an *existing* cluster +# Try to join an existing cluster /usr/local/bin/etcd \ --name ${NAME} \ --debug \ --log-package-levels etcdmain=DEBUG,etcdserver=DEBUG \ + --data-dir $MOUNTPOINT \ --initial-advertise-peer-urls http://${IP}:2380 \ --listen-peer-urls http://${IP}:2380 \ --listen-client-urls http://${IP}:2379,http://127.0.0.1:2379 \ diff --git a/projects/demo/etcd/etcd.yml b/projects/demo/etcd/etcd.yml index 1a553bd8a..8e4810d22 100644 --- a/projects/demo/etcd/etcd.yml +++ b/projects/demo/etcd/etcd.yml @@ -2,7 +2,9 @@ kernel: image: "mobylinux/kernel:4.9.x" cmdline: "console=ttyS0 console=tty0 page_poison=1" init: - - "mobylinux/init:a27e32a8d6c8865d691fbfb4d0bbb93846cf7802" + - mobylinux/init:925c88f42d92d57cd36b656db1f8757b152163a7 + - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9 + - mobylinux/containerd:68bb523deea09da293d675cbf88474eced540b8c onboot: - name: sysctl image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c" @@ -12,6 +14,13 @@ onboot: capabilities: - CAP_SYS_ADMIN readonly: true + - name: format + image: "mobylinux/format:53748000acf515549d398e6ae68545c26c0f3a2e" + binds: + - /dev:/dev + capabilities: + - CAP_SYS_ADMIN + - CAP_MKNOD services: - name: rngd image: "mobylinux/rngd:3dad6dd43270fa632ac031e99d1947f20b22eec9@sha256:1c93c1db7196f6f71f8e300bc1d15f0376dd18e8891c8789d77c8ff19f3a9a92" @@ -33,12 +42,15 @@ services: - name: etcd image: "mobylinux/etcd" capabilities: - - CAP_NET_BIND_SERVICE - CAP_CHOWN - CAP_SETUID - CAP_SETGID - CAP_DAC_OVERRIDE + - CAP_SYS_ADMIN + - CAP_MKNOD net: host + binds: + - /dev:/dev outputs: - format: kernel+initrd # - format: gcp diff --git a/projects/demo/etcd/infrakit.json b/projects/demo/etcd/infrakit.json index 412333065..0d7cef887 100644 --- a/projects/demo/etcd/infrakit.json +++ b/projects/demo/etcd/infrakit.json @@ -14,7 +14,7 @@ "Plugin": "instance-hyperkit", "Properties": { "Moby": "etcd", - "Disk" : 0, + "Disk" : 1024, "CPUs" : 1, "Memory" : 1024 } diff --git a/projects/demo/etcd/start-infrakit.sh b/projects/demo/etcd/start-infrakit.sh index 0cf4162f2..d0c37e2ae 100755 --- a/projects/demo/etcd/start-infrakit.sh +++ b/projects/demo/etcd/start-infrakit.sh @@ -6,6 +6,7 @@ INFRAKIT_HOME=~/.infrakit IK_PLUGINS=$INFRAKIT_HOME/plugins rm -rf $INFRAKIT_HOME +mkdir -p $INFRAKIT_HOME/cli infrakit-flavor-vanilla & infrakit-instance-hyperkit & diff --git a/projects/demo/intro/intro.yml b/projects/demo/intro/intro.yml index 02160ee93..810f9b12f 100644 --- a/projects/demo/intro/intro.yml +++ b/projects/demo/intro/intro.yml @@ -2,7 +2,9 @@ kernel: image: "mobylinux/kernel:4.9.x" cmdline: "console=ttyS0 console=tty0 page_poison=1" init: - - "mobylinux/init:a27e32a8d6c8865d691fbfb4d0bbb93846cf7802" + - mobylinux/init:925c88f42d92d57cd36b656db1f8757b152163a7 + - mobylinux/runc:b0fb122e10dbb7e4e45115177a61a3f8d68c19a9 + - mobylinux/containerd:68bb523deea09da293d675cbf88474eced540b8c onboot: - name: sysctl image: "mobylinux/sysctl:2cf2f9d5b4d314ba1bfc22b2fe931924af666d8c" diff --git a/src/cmd/infrakit-instance-hyperkit/instance.go b/src/cmd/infrakit-instance-hyperkit/instance.go index 6b1d4d1f7..79039a129 100644 --- a/src/cmd/infrakit-instance-hyperkit/instance.go +++ b/src/cmd/infrakit-instance-hyperkit/instance.go @@ -21,6 +21,7 @@ func NewHyperKitPlugin(vmDir, hyperkit, vpnkitSock string) instance.Plugin { return &hyperkitPlugin{VMDir: vmDir, HyperKit: hyperkit, VPNKitSock: vpnkitSock, + DiskDir: path.Join(vmDir, "disks"), } } @@ -33,6 +34,9 @@ type hyperkitPlugin struct { // VPNKitSock is the path to the VPNKit Unix domain socket. VPNKitSock string + + // DiskDir is the path to persistent (across reboots) disk images + DiskDir string } // Validate performs local validation on a provision request. @@ -60,8 +64,9 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) { if properties["Memory"] == nil { properties["Memory"] = 512 } - if properties["Disk"] == nil { - properties["Disk"] = 256 + diskSize := 0 + if properties["Disk"] != nil { + diskSize = int(properties["Disk"].(float64)) } instanceDir, err := ioutil.TempDir(p.VMDir, "infrakit-") @@ -71,12 +76,15 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) { id := instance.ID(path.Base(instanceDir)) log.Infof("[%s] New instance", id) - // The LogicalID may be a IP address. If so, translate it into a - // magic UUID which cause VPNKit to assign a fixed IP address logicalID := string(id) uuidStr := "" + + diskImage := "" if spec.LogicalID != nil { logicalID = string(*spec.LogicalID) + // The LogicalID may be a IP address. If so, translate + // it into a magic UUID which cause VPNKit to assign a + // fixed IP address if ip := net.ParseIP(logicalID); len(ip) > 0 { uuid := make([]byte, 16) uuid[12] = ip.To4()[0] @@ -85,12 +93,23 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) { uuid[15] = ip.To4()[3] uuidStr = fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]) } + // If a LogicalID is supplied and the Disk size is + // non-zero, we place the disk in a special directory + // so it persists across reboots. + if diskSize != 0 { + diskImage = path.Join(p.DiskDir, logicalID+".img") + // Make sure the directory exists + err = os.MkdirAll(p.DiskDir, 0755) + if err != nil { + return nil, err + } + } } log.Infof("[%s] LogicalID: %s", id, logicalID) log.Debugf("[%s] UUID: %s", id, uuidStr) // Start a HyperKit instance - h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, "") + h, err := hyperkit.New(p.HyperKit, instanceDir, p.VPNKitSock, diskImage) if err != nil { return nil, err } @@ -98,12 +117,12 @@ func (p hyperkitPlugin) Provision(spec instance.Spec) (*instance.ID, error) { h.Initrd = properties["Moby"].(string) + "-initrd.img" h.CPUs = int(properties["CPUs"].(float64)) h.Memory = int(properties["Memory"].(float64)) - h.DiskSize = int(properties["Disk"].(float64)) + h.DiskSize = diskSize h.UUID = uuidStr h.UserData = spec.Init h.Console = hyperkit.ConsoleFile log.Infof("[%s] Booting: %s/%s", id, h.Kernel, h.Initrd) - log.Infof("[%s] %d CPUs, %dMB Memory, %dMB Disk", id, h.CPUs, h.Memory, h.DiskSize) + log.Infof("[%s] %d CPUs, %dMB Memory, %dMB Disk (%s)", id, h.CPUs, h.Memory, h.DiskSize, h.DiskImage) err = h.Start("console=ttyS0") if err != nil {