From 35c84cd0e637660f7518cd9bed20c5991f9f82e7 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 27 Jun 2017 16:35:25 +0100 Subject: [PATCH 1/9] swarmd: Sync init with linuxkit.yml Signed-off-by: Ian Campbell --- projects/swarmd/swarmd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index 460aaf37a..a7b8b2fbd 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -2,7 +2,7 @@ kernel: image: "linuxkit/kernel:4.9.35" cmdline: "console=ttyS0 console=tty0 page_poison=1" init: - - linuxkit/init:12348442d56c2ee9abf13ff38dff2e36b515bd1e + - linuxkit/init:24942921d1356bb801b30ca6d7197d2bfdcc26f9 - linuxkit/runc:2310ad9d266cf5d9c4d07613bd2135ed7eb8a21f - linuxkit/containerd:e0607d117e0286792c5bd62d9a7e2a9c49be3bbf - linuxkit/ca-certificates:67acf038c44bb191ebb704ec7bb39a1524052cdf From fc5d166eb5cd1a35acef454a88b5418e4674d508 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 27 Jun 2017 16:38:57 +0100 Subject: [PATCH 2/9] swarmd: Add a getty container, drop console=tty0 Signed-off-by: Ian Campbell --- projects/swarmd/swarmd.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index a7b8b2fbd..111b97d1e 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -1,6 +1,6 @@ kernel: image: "linuxkit/kernel:4.9.35" - cmdline: "console=ttyS0 console=tty0 page_poison=1" + cmdline: "console=ttyS0 page_poison=1" init: - linuxkit/init:24942921d1356bb801b30ca6d7197d2bfdcc26f9 - linuxkit/runc:2310ad9d266cf5d9c4d07613bd2135ed7eb8a21f @@ -22,6 +22,10 @@ onboot: - name: metadata image: "linuxkit/metadata:4e73345cdcb4f7e9df07b0ee7aede652960297f2" services: + - name: getty + image: "linuxkit/getty:9f27c1272b6d128c9a09745e916f151d09cb0d27" + env: + - INSECURE=true - name: qemu-ga image: "linuxkit/qemu-ga:585e4f0161a4df7583d5e0479d7621040c1ee140" binds: From 415bdd396c804ac517056f121829171d9e5b2933 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 12:57:02 +0100 Subject: [PATCH 3/9] swarmd: Use common package.mk to simplify Makefile Signed-off-by: Ian Campbell --- projects/swarmd/swarmd/Makefile | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/projects/swarmd/swarmd/Makefile b/projects/swarmd/swarmd/Makefile index 13ae4dd66..b8e0bb5f7 100644 --- a/projects/swarmd/swarmd/Makefile +++ b/projects/swarmd/swarmd/Makefile @@ -1,15 +1,4 @@ -.PHONY: tag push -all: push - -ORG?=linuxkit IMAGE=swarmd +NETWORK=1 -HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') - -default: push - -tag: Dockerfile - docker build --no-cache -t $(ORG)/$(IMAGE):$(HASH) . - -push: tag - docker pull $(ORG)/$(IMAGE):$(HASH) || docker push $(ORG)/$(IMAGE):$(HASH) +include ../../../pkg/package.mk From 974b622c0601464328c652f4f6d10ca95e54cc3d Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 13:16:24 +0100 Subject: [PATCH 4/9] swarmd: Update to latest WIP code This version is more up to date and works with the current containerd packages. swarmd needs /tmp to share /tmp/containerd with containerd. Signed-off-by: Ian Campbell --- projects/swarmd/swarmd.yml | 3 ++- projects/swarmd/swarmd/Dockerfile | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index 111b97d1e..960a1fe3b 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -35,12 +35,13 @@ services: - name: ntpd image: "linuxkit/openntpd:19370f5d9bec84eb91073b7196b732f1301d9c90" - name: swarmd - image: "linuxkit/swarmd:1002422b78339a767559058d704b086889e90447" + image: "linuxkit/swarmd:a65dc4a5464a56771af056387b85d3d4593dbcf0" command: ["/usr/bin/swarmd", "--containerd-addr=/run/containerd/containerd.sock", "--log-level=debug", "--state-dir=/var/lib/swarmd"] capabilities: - all pid: host binds: + - /tmp:/tmp - /run/containerd/containerd.sock:/run/containerd/containerd.sock - /var/lib/containerd:/var/lib/containerd - /var/lib/swarmd:/var/lib/swarmd diff --git a/projects/swarmd/swarmd/Dockerfile b/projects/swarmd/swarmd/Dockerfile index f07908c7f..9733df026 100644 --- a/projects/swarmd/swarmd/Dockerfile +++ b/projects/swarmd/swarmd/Dockerfile @@ -16,7 +16,7 @@ ENV GOPATH=/root/go # https://github.com/ijc25/swarmkit/tree/containerd-wip ENV SWARMKIT_REPO=https://github.com/ijc25/swarmkit ENV SWARMKIT_BRANCH=containerd-wip -ENV SWARMKIT_COMMIT=cb429323fa86df0c279a8d629e674f79a20e01e3 +ENV SWARMKIT_COMMIT=4a484ccb498bee117fe6167d5a5e7ea0f6d4f2e9 RUN mkdir -p $GOPATH/src/github.com/docker && \ cd $GOPATH/src/github.com/docker && \ From dcb40158c6f02dcbc9f6fd32249cc3d0df50c81e Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 29 Jun 2017 15:32:59 +0100 Subject: [PATCH 5/9] swarmd: Remove APK residuals from output image Signed-off-by: Ian Campbell --- projects/swarmd/swarmd/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/swarmd/swarmd/Dockerfile b/projects/swarmd/swarmd/Dockerfile index 9733df026..005ce7914 100644 --- a/projects/swarmd/swarmd/Dockerfile +++ b/projects/swarmd/swarmd/Dockerfile @@ -36,6 +36,9 @@ RUN apk add --no-cache --initdb -p /out \ musl \ util-linux +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + RUN mkdir -p /out/usr/bin/ /out/etc RUN cp bin/swarmd bin/swarmctl /out/usr/bin/ RUN strip /out/usr/bin/swarmd /out/usr/bin/swarmctl From 0277df421dc3d5ae149dcbee79352e16cbd616df Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 11 May 2017 19:13:40 +0100 Subject: [PATCH 6/9] swarmd: support networking via Weave and the CNI reference plugins Signed-off-by: Ian Campbell --- projects/swarmd/swarmd.yml | 26 +++++++++++++ projects/swarmd/swarmd/Dockerfile | 62 ++++++++++++++++++++++++------- 2 files changed, 75 insertions(+), 13 deletions(-) diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index 960a1fe3b..9d97644e2 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -34,6 +34,16 @@ services: image: "linuxkit/rngd:1516d5d70683a5d925fe475eb1b6164a2f67ac3b" - name: ntpd image: "linuxkit/openntpd:19370f5d9bec84eb91073b7196b732f1301d9c90" + - name: weave + image: weaveworks/weave@sha256:05172329b6ff72099db7bb891ac311b89948a3064ca9b8641c6b4abe38548677 # Must match swarmd/Dockerfile + command: ["/bin/sh", "/home/weave/weaver-wrapper"] + capabilities: + - all + pid: host + binds: + - /usr/local/bin/weaver-wrapper:/home/weave/weaver-wrapper + - /var:/var + - /var/lib/swarmd:/weavedb - name: swarmd image: "linuxkit/swarmd:a65dc4a5464a56771af056387b85d3d4593dbcf0" command: ["/usr/bin/swarmd", "--containerd-addr=/run/containerd/containerd.sock", "--log-level=debug", "--state-dir=/var/lib/swarmd"] @@ -49,3 +59,19 @@ services: files: - path: /etc/sysctl.d/01-swarmd.conf contents: 'net.ipv4.ip_forward = 1' + - path: usr/local/bin/weaver-wrapper + contents: | + /home/weave/weaver \ + --weave-bridge=weave \ + --datapath=datapath \ + --docker-api="" \ + --ipalloc-range=192.168.106.0/24 \ + --http-addr=127.0.0.1:6784 \ + --status-addr=127.0.0.1:6782 & + + while [ ! -e /sys/class/net/weave/ ] ; do + sleep 1s + done + + /home/weave/weave --local expose net:default + wait %1 diff --git a/projects/swarmd/swarmd/Dockerfile b/projects/swarmd/swarmd/Dockerfile index 005ce7914..b7fb8f948 100644 --- a/projects/swarmd/swarmd/Dockerfile +++ b/projects/swarmd/swarmd/Dockerfile @@ -1,18 +1,44 @@ +FROM weaveworks/weave@sha256:05172329b6ff72099db7bb891ac311b89948a3064ca9b8641c6b4abe38548677 AS weave + +# Nothing to do in here, just for COPY --from=weave below + FROM linuxkit/alpine:9bcf61f605ef0ce36cc94d59b8eac307862de6e1 AS build RUN \ apk update && apk upgrade && \ apk add --no-cache \ + bash \ ca-certificates \ gcc \ git \ go \ libc-dev \ + linux-headers \ make \ && true ENV GOPATH=/root/go +##################################################################### +# Output filesystem skeleton + +RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ +RUN apk add --no-cache --initdb -p /out \ + alpine-baselayout \ + busybox \ + ca-certificates \ + iptables \ + musl \ + util-linux + +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + +RUN mkdir -p /out/usr/bin/ /out/etc /out/opt/cni/bin /out/etc/cni/net.d + +##################################################################### +# Swarmd + # https://github.com/ijc25/swarmkit/tree/containerd-wip ENV SWARMKIT_REPO=https://github.com/ijc25/swarmkit ENV SWARMKIT_BRANCH=containerd-wip @@ -27,22 +53,32 @@ RUN git checkout $SWARMKIT_COMMIT RUN make binaries GO_GCFLAGS="-buildmode pie --ldflags '-extldflags \"-fno-PIC -static\"'" -RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ -RUN apk add --no-cache --initdb -p /out \ - alpine-baselayout \ - busybox \ - ca-certificates \ - iptables \ - musl \ - util-linux - -# Remove apk residuals. We have a read-only rootfs, so apk is of no use. -RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache - -RUN mkdir -p /out/usr/bin/ /out/etc RUN cp bin/swarmd bin/swarmctl /out/usr/bin/ RUN strip /out/usr/bin/swarmd /out/usr/bin/swarmctl +##################################################################### +# CNI + +ENV CNI_REPO=https://github.com/containernetworking/cni +ENV CNI_COMMIT=v0.5.2 + +RUN mkdir -p $GOPATH/src/github.com/containernetworking && \ + cd $GOPATH/src/github.com/containernetworking && \ + git clone $CNI_REPO +WORKDIR $GOPATH/src/github.com/containernetworking/cni +RUN [ -z "$CNI_BRANCH" ] || git fetch origin $CNI_BRANCH +RUN git checkout $CNI_COMMIT + +RUN ./build.sh -buildmode pie --ldflags '-extldflags "-fno-PIC -static"' + +RUN cp bin/bridge bin/host-local bin/dhcp /out/opt/cni/bin/ + +##################################################################### +# Weave + +COPY --from=weave /usr/bin/weaveutil /out/opt/cni/bin/weave-net +RUN ln -s weave-net /out/opt/cni/bin/weave-ipam + FROM scratch WORKDIR / ENTRYPOINT [] From ee61fa03df59cb510a11c41808a8f50bead3adea Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Thu, 29 Jun 2017 15:22:17 +0100 Subject: [PATCH 7/9] swarmd: Move to linuxkitprojects hub org Signed-off-by: Ian Campbell --- projects/swarmd/swarmd/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/projects/swarmd/swarmd/Makefile b/projects/swarmd/swarmd/Makefile index b8e0bb5f7..9550aeffe 100644 --- a/projects/swarmd/swarmd/Makefile +++ b/projects/swarmd/swarmd/Makefile @@ -1,3 +1,4 @@ +ORG?=linuxkitprojects IMAGE=swarmd NETWORK=1 From 4437fe84e28aa99b73ae89400aeb819ed91b07d8 Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 15:52:40 +0100 Subject: [PATCH 8/9] swarmd: Update tag after recent updates. Signed-off-by: Ian Campbell --- projects/swarmd/swarmd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index 9d97644e2..0f1b39b2e 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -45,7 +45,7 @@ services: - /var:/var - /var/lib/swarmd:/weavedb - name: swarmd - image: "linuxkit/swarmd:a65dc4a5464a56771af056387b85d3d4593dbcf0" + image: "linuxkitprojects/swarmd:1cd4c061cc7327750d2a12c267db6d4d9e26b1d3" command: ["/usr/bin/swarmd", "--containerd-addr=/run/containerd/containerd.sock", "--log-level=debug", "--state-dir=/var/lib/swarmd"] capabilities: - all From f0ca8cdabdf0410480ca2fa3f6b6f857ed82ea6d Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Wed, 28 Jun 2017 14:22:54 +0100 Subject: [PATCH 9/9] swarmd: Update README.md Signed-off-by: Ian Campbell --- projects/swarmd/README.md | 32 ++++++++++++++++++------------- projects/swarmd/swarmd/Dockerfile | 2 +- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/projects/swarmd/README.md b/projects/swarmd/README.md index 0fe0d7cb7..03cb866a9 100644 --- a/projects/swarmd/README.md +++ b/projects/swarmd/README.md @@ -3,27 +3,33 @@ This adds a `swarmd` package for Moby which contains the standalone swarmkit orchestration daemon (`swarmd`) and CLI tool (`swarmctl`). -The package tracks [docker/swarmkit#1965][PR1965] which -is a WIP PR adding a containerd executor to swarmkit. +The package tracks [ijc's `container-wip` branch][containerd-wip]. +Compared with mainline swarmkit (which container a basic containerd +executor merged in [PR1965]) this reworks the executor to use the +container client library and adds support for CNI networking. -With a suitable moby image (such as `swarmd.yml` from this directory) -something like this should work: +With a suitable LinuxKit image (such as `swarmd.yml` from this +directory) something like this should work: - runc exec swarmd swarmctl service create --image docker.io/library/nginx:alpine --name nginx - runc exec swarmd swarmctl service ls + ctr exec -- swarmd swarmd swarmctl service create --image docker.io/library/nginx:alpine --name nginx + ctr exec -- swarmd swarmd swarmctl service ls + +Note that `swarmd` uses the "swarmd" containerd namespace, so to see +swarmd managed containers you will need to use `-n swarmd` on all +`ctr` commands e.g.: + + ctr -n swarmd containers ls + +Alternatively you may export `CONTAINERD_NAMESPACE=swarmd`. ### TODO -Currently the swarm state directory needs to be at a path which is -identical from the PoV of both the `containerd` and `swarmd` -processes. For now this means that the swarmkit state is put in -`/var/lib/containerd/swarmd`. - -Bootstrapping a cluster needs more invesigation. Tokens and join +Bootstrapping a cluster needs more investigation. Tokens and join addresses can currently only be passed on the `swarmd` command line which is inconvenient for automated image deployment. Swarmkit [PR 1965][PR1965] also contains a number of TODOs which are not separately listed here. -[PR1665]: https://github.com/docker/swarmkit/pull/1965 +[PR1965]: https://github.com/docker/swarmkit/pull/1965 +[containerd-wip]: https://github.com/ijc/swarmkit/tree/containerd-wip diff --git a/projects/swarmd/swarmd/Dockerfile b/projects/swarmd/swarmd/Dockerfile index b7fb8f948..749291519 100644 --- a/projects/swarmd/swarmd/Dockerfile +++ b/projects/swarmd/swarmd/Dockerfile @@ -39,7 +39,7 @@ RUN mkdir -p /out/usr/bin/ /out/etc /out/opt/cni/bin /out/etc/cni/net.d ##################################################################### # Swarmd -# https://github.com/ijc25/swarmkit/tree/containerd-wip +# https://github.com/ijc/swarmkit/tree/containerd-wip ENV SWARMKIT_REPO=https://github.com/ijc25/swarmkit ENV SWARMKIT_BRANCH=containerd-wip ENV SWARMKIT_COMMIT=4a484ccb498bee117fe6167d5a5e7ea0f6d4f2e9