diff --git a/projects/swarmd/README.md b/projects/swarmd/README.md index 0fe0d7cb7..03cb866a9 100644 --- a/projects/swarmd/README.md +++ b/projects/swarmd/README.md @@ -3,27 +3,33 @@ This adds a `swarmd` package for Moby which contains the standalone swarmkit orchestration daemon (`swarmd`) and CLI tool (`swarmctl`). -The package tracks [docker/swarmkit#1965][PR1965] which -is a WIP PR adding a containerd executor to swarmkit. +The package tracks [ijc's `container-wip` branch][containerd-wip]. +Compared with mainline swarmkit (which container a basic containerd +executor merged in [PR1965]) this reworks the executor to use the +container client library and adds support for CNI networking. -With a suitable moby image (such as `swarmd.yml` from this directory) -something like this should work: +With a suitable LinuxKit image (such as `swarmd.yml` from this +directory) something like this should work: - runc exec swarmd swarmctl service create --image docker.io/library/nginx:alpine --name nginx - runc exec swarmd swarmctl service ls + ctr exec -- swarmd swarmd swarmctl service create --image docker.io/library/nginx:alpine --name nginx + ctr exec -- swarmd swarmd swarmctl service ls + +Note that `swarmd` uses the "swarmd" containerd namespace, so to see +swarmd managed containers you will need to use `-n swarmd` on all +`ctr` commands e.g.: + + ctr -n swarmd containers ls + +Alternatively you may export `CONTAINERD_NAMESPACE=swarmd`. ### TODO -Currently the swarm state directory needs to be at a path which is -identical from the PoV of both the `containerd` and `swarmd` -processes. For now this means that the swarmkit state is put in -`/var/lib/containerd/swarmd`. - -Bootstrapping a cluster needs more invesigation. Tokens and join +Bootstrapping a cluster needs more investigation. Tokens and join addresses can currently only be passed on the `swarmd` command line which is inconvenient for automated image deployment. Swarmkit [PR 1965][PR1965] also contains a number of TODOs which are not separately listed here. -[PR1665]: https://github.com/docker/swarmkit/pull/1965 +[PR1965]: https://github.com/docker/swarmkit/pull/1965 +[containerd-wip]: https://github.com/ijc/swarmkit/tree/containerd-wip diff --git a/projects/swarmd/swarmd.yml b/projects/swarmd/swarmd.yml index 460aaf37a..0f1b39b2e 100644 --- a/projects/swarmd/swarmd.yml +++ b/projects/swarmd/swarmd.yml @@ -1,8 +1,8 @@ kernel: image: "linuxkit/kernel:4.9.35" - cmdline: "console=ttyS0 console=tty0 page_poison=1" + cmdline: "console=ttyS0 page_poison=1" init: - - linuxkit/init:12348442d56c2ee9abf13ff38dff2e36b515bd1e + - linuxkit/init:24942921d1356bb801b30ca6d7197d2bfdcc26f9 - linuxkit/runc:2310ad9d266cf5d9c4d07613bd2135ed7eb8a21f - linuxkit/containerd:e0607d117e0286792c5bd62d9a7e2a9c49be3bbf - linuxkit/ca-certificates:67acf038c44bb191ebb704ec7bb39a1524052cdf @@ -22,6 +22,10 @@ onboot: - name: metadata image: "linuxkit/metadata:4e73345cdcb4f7e9df07b0ee7aede652960297f2" services: + - name: getty + image: "linuxkit/getty:9f27c1272b6d128c9a09745e916f151d09cb0d27" + env: + - INSECURE=true - name: qemu-ga image: "linuxkit/qemu-ga:585e4f0161a4df7583d5e0479d7621040c1ee140" binds: @@ -30,13 +34,24 @@ services: image: "linuxkit/rngd:1516d5d70683a5d925fe475eb1b6164a2f67ac3b" - name: ntpd image: "linuxkit/openntpd:19370f5d9bec84eb91073b7196b732f1301d9c90" + - name: weave + image: weaveworks/weave@sha256:05172329b6ff72099db7bb891ac311b89948a3064ca9b8641c6b4abe38548677 # Must match swarmd/Dockerfile + command: ["/bin/sh", "/home/weave/weaver-wrapper"] + capabilities: + - all + pid: host + binds: + - /usr/local/bin/weaver-wrapper:/home/weave/weaver-wrapper + - /var:/var + - /var/lib/swarmd:/weavedb - name: swarmd - image: "linuxkit/swarmd:1002422b78339a767559058d704b086889e90447" + image: "linuxkitprojects/swarmd:1cd4c061cc7327750d2a12c267db6d4d9e26b1d3" command: ["/usr/bin/swarmd", "--containerd-addr=/run/containerd/containerd.sock", "--log-level=debug", "--state-dir=/var/lib/swarmd"] capabilities: - all pid: host binds: + - /tmp:/tmp - /run/containerd/containerd.sock:/run/containerd/containerd.sock - /var/lib/containerd:/var/lib/containerd - /var/lib/swarmd:/var/lib/swarmd @@ -44,3 +59,19 @@ services: files: - path: /etc/sysctl.d/01-swarmd.conf contents: 'net.ipv4.ip_forward = 1' + - path: usr/local/bin/weaver-wrapper + contents: | + /home/weave/weaver \ + --weave-bridge=weave \ + --datapath=datapath \ + --docker-api="" \ + --ipalloc-range=192.168.106.0/24 \ + --http-addr=127.0.0.1:6784 \ + --status-addr=127.0.0.1:6782 & + + while [ ! -e /sys/class/net/weave/ ] ; do + sleep 1s + done + + /home/weave/weave --local expose net:default + wait %1 diff --git a/projects/swarmd/swarmd/Dockerfile b/projects/swarmd/swarmd/Dockerfile index f07908c7f..749291519 100644 --- a/projects/swarmd/swarmd/Dockerfile +++ b/projects/swarmd/swarmd/Dockerfile @@ -1,31 +1,26 @@ +FROM weaveworks/weave@sha256:05172329b6ff72099db7bb891ac311b89948a3064ca9b8641c6b4abe38548677 AS weave + +# Nothing to do in here, just for COPY --from=weave below + FROM linuxkit/alpine:9bcf61f605ef0ce36cc94d59b8eac307862de6e1 AS build RUN \ apk update && apk upgrade && \ apk add --no-cache \ + bash \ ca-certificates \ gcc \ git \ go \ libc-dev \ + linux-headers \ make \ && true ENV GOPATH=/root/go -# https://github.com/ijc25/swarmkit/tree/containerd-wip -ENV SWARMKIT_REPO=https://github.com/ijc25/swarmkit -ENV SWARMKIT_BRANCH=containerd-wip -ENV SWARMKIT_COMMIT=cb429323fa86df0c279a8d629e674f79a20e01e3 - -RUN mkdir -p $GOPATH/src/github.com/docker && \ - cd $GOPATH/src/github.com/docker && \ - git clone $SWARMKIT_REPO -WORKDIR $GOPATH/src/github.com/docker/swarmkit -RUN [ -z "$SWARMKIT_BRANCH" ] || git fetch origin $SWARMKIT_BRANCH -RUN git checkout $SWARMKIT_COMMIT - -RUN make binaries GO_GCFLAGS="-buildmode pie --ldflags '-extldflags \"-fno-PIC -static\"'" +##################################################################### +# Output filesystem skeleton RUN mkdir -p /out/etc/apk && cp -r /etc/apk/* /out/etc/apk/ RUN apk add --no-cache --initdb -p /out \ @@ -36,10 +31,54 @@ RUN apk add --no-cache --initdb -p /out \ musl \ util-linux -RUN mkdir -p /out/usr/bin/ /out/etc +# Remove apk residuals. We have a read-only rootfs, so apk is of no use. +RUN rm -rf /out/etc/apk /out/lib/apk /out/var/cache + +RUN mkdir -p /out/usr/bin/ /out/etc /out/opt/cni/bin /out/etc/cni/net.d + +##################################################################### +# Swarmd + +# https://github.com/ijc/swarmkit/tree/containerd-wip +ENV SWARMKIT_REPO=https://github.com/ijc25/swarmkit +ENV SWARMKIT_BRANCH=containerd-wip +ENV SWARMKIT_COMMIT=4a484ccb498bee117fe6167d5a5e7ea0f6d4f2e9 + +RUN mkdir -p $GOPATH/src/github.com/docker && \ + cd $GOPATH/src/github.com/docker && \ + git clone $SWARMKIT_REPO +WORKDIR $GOPATH/src/github.com/docker/swarmkit +RUN [ -z "$SWARMKIT_BRANCH" ] || git fetch origin $SWARMKIT_BRANCH +RUN git checkout $SWARMKIT_COMMIT + +RUN make binaries GO_GCFLAGS="-buildmode pie --ldflags '-extldflags \"-fno-PIC -static\"'" + RUN cp bin/swarmd bin/swarmctl /out/usr/bin/ RUN strip /out/usr/bin/swarmd /out/usr/bin/swarmctl +##################################################################### +# CNI + +ENV CNI_REPO=https://github.com/containernetworking/cni +ENV CNI_COMMIT=v0.5.2 + +RUN mkdir -p $GOPATH/src/github.com/containernetworking && \ + cd $GOPATH/src/github.com/containernetworking && \ + git clone $CNI_REPO +WORKDIR $GOPATH/src/github.com/containernetworking/cni +RUN [ -z "$CNI_BRANCH" ] || git fetch origin $CNI_BRANCH +RUN git checkout $CNI_COMMIT + +RUN ./build.sh -buildmode pie --ldflags '-extldflags "-fno-PIC -static"' + +RUN cp bin/bridge bin/host-local bin/dhcp /out/opt/cni/bin/ + +##################################################################### +# Weave + +COPY --from=weave /usr/bin/weaveutil /out/opt/cni/bin/weave-net +RUN ln -s weave-net /out/opt/cni/bin/weave-ipam + FROM scratch WORKDIR / ENTRYPOINT [] diff --git a/projects/swarmd/swarmd/Makefile b/projects/swarmd/swarmd/Makefile index 13ae4dd66..9550aeffe 100644 --- a/projects/swarmd/swarmd/Makefile +++ b/projects/swarmd/swarmd/Makefile @@ -1,15 +1,5 @@ -.PHONY: tag push -all: push - -ORG?=linuxkit +ORG?=linuxkitprojects IMAGE=swarmd +NETWORK=1 -HASH?=$(shell git ls-tree HEAD -- ../$(notdir $(CURDIR)) | awk '{print $$3}') - -default: push - -tag: Dockerfile - docker build --no-cache -t $(ORG)/$(IMAGE):$(HASH) . - -push: tag - docker pull $(ORG)/$(IMAGE):$(HASH) || docker push $(ORG)/$(IMAGE):$(HASH) +include ../../../pkg/package.mk