Merge pull request #1201 from justincormack/newtools

Add new minimal init prototype and new tools

base/containerd/Dockerfile

@@ -7,13 +7,13 @@ RUN \
   libc-dev \
   make \
   && true
-ENV CONTAINERD_COMMIT=3b79682548339895fcf9976f60ddea8abc5fc97e
+ENV CONTAINERD_COMMIT=1dc5d652ac1adf8f0a92ee8eff7af07b129d4e21
 RUN mkdir -p $GOPATH/src/github.com/docker && \
   cd $GOPATH/src/github.com/docker && \
   git clone https://github.com/docker/containerd.git
 WORKDIR $GOPATH/src/github.com/docker/containerd
 RUN git checkout $CONTAINERD_COMMIT
 RUN make binaries GO_GCFLAGS="-buildmode pie --ldflags '-extldflags \"-fno-PIC -static\"'"
-RUN cp bin/containerd bin/ctr bin/containerd-shim /usr/bin/
+RUN cp bin/containerd bin/ctr bin/containerd-shim bin/dist /usr/bin/
 WORKDIR /
 COPY . .

base/init/.gitignore

@@ -0,0 +1,2 @@
+sbin/
+usr/

base/init/Dockerfile

@@ -0,0 +1,11 @@
+FROM alpine:3.5
+
+RUN \
+  apk update && apk upgrade -a && \
+  apk add --no-cache \
+  dhcpcd \
+  e2fsprogs \
+  e2fsprogs-extra \
+  && true
+
+COPY . ./

base/init/Makefile

@@ -0,0 +1,52 @@
+CONTAINERD_IMAGE=mobylinux/containerd:6fac214f15f3dffc1c77d82874d828483b5ff0d0@sha256:59f9b78506a00c77c0fbde857a9dceb7812b9f4387c5d09ddc751ef7a42ae632
+CONTAINERD_BINARIES=usr/bin/containerd usr/bin/containerd-shim usr/bin/ctr usr/bin/dist
+
+RUNC_IMAGE=mobylinux/runc:e6426f67d17d7e34dceded8b1ab45454b41652e2@sha256:fe2014f51c18e2b0ef48c3ddfe7fac0d35547c59de5f3bc32b10c70ed123cdc6
+RUNC_BINARY=usr/bin/runc
+
+C_COMPILE=mobylinux/c-compile:81a6bd8ff45d769b60a2ee1acdaccda11ab835c8@sha256:eac250997a3b9784d3285a03c0c8311d4ca6fb63dc75164c987411ba93006487
+START_STOP_DAEMON=sbin/start-stop-daemon
+
+default: push
+
+$(RUNC_BINARY):
+	mkdir -p $(dir $@)
+	docker run --rm --net=none $(RUNC_IMAGE) tar cf - $@ | tar xf -
+
+$(CONTAINERD_BINARIES):
+	mkdir -p $(dir $@)
+	docker run --rm --net=none $(CONTAINERD_IMAGE) tar cf - $@ | tar xf -
+
+$(START_STOP_DAEMON): start-stop-daemon.c
+	mkdir -p $(dir $@)
+	tar cf - $^ | docker run --rm --net=none --log-driver=none -i $(C_COMPILE) -o $@ | tar xf -
+
+.PHONY: tag push
+
+BASE=alpine:3.5
+IMAGE=init
+
+ETC=$(shell find etc -type f)
+
+hash: Dockerfile $(ETC) init $(RUNC_BINARY) $(CONTAINERD_BINARIES) $(START_STOP_DAEMON)
+	DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
+	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
+	docker run --rm $(IMAGE):build sh -c 'cat $^ /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@
+
+push: hash
+	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+		(docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
+		 docker push mobylinux/$(IMAGE):$(shell cat hash))
+	docker rmi $(IMAGE):build
+	rm -f hash
+
+tag: hash
+	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+		docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
+	docker rmi $(IMAGE):build
+	rm -f hash
+
+clean:
+	rm -rf hash sbin usr
+
+.DELETE_ON_ERROR:

base/init/etc/dhcpcd.conf

@@ -0,0 +1,46 @@
+# Moby dhcpcd config
+
+# Only configure standard external ethernet
+allowinterfaces eth*
+
+# Inform the DHCP server of our hostname for DDNS.
+hostname
+
+# Use the hardware address of the interface for the Client ID.
+clientid
+# or
+# Use the same DUID + IAID as set in DHCPv6 for DHCPv4 ClientID as per RFC4361.
+# Some non-RFC compliant DHCP servers do not reply with this set.
+# In this case, comment out duid and enable clientid above.
+#duid
+
+# Persist interface configuration when dhcpcd exits.
+persistent
+
+# Rapid commit support.
+# Safe to enable by default because it requires the equivalent option set
+# on the server to actually work.
+option rapid_commit
+
+# A list of options to request from the DHCP server.
+option domain_name_servers, domain_name, domain_search, host_name
+option classless_static_routes
+# Most distributions have NTP support.
+option ntp_servers
+# Respect the network MTU. This is applied to DHCP routes.
+option interface_mtu
+
+# A ServerID is required by RFC2131.
+require dhcp_server_identifier
+
+# Generate Stable Private IPv6 Addresses instead of hardware based ones
+slaac private
+
+# Do not wait
+nodelay
+
+# Do not arp to check IP
+noarp
+
+# Only fork when we have ipv4
+# waitip 4

base/init/etc/init.d/containerd

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# bring up containerd
+ulimit -n 1048576
+ulimit -p unlimited
+
+printf "\nStarting containerd\n"
+mkdir -p /var/log
+/sbin/start-stop-daemon --start --exec /usr/bin/containerd

base/init/etc/init.d/containers

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+# TODO more robust
+while [ ! -S /run/containerd/containerd.sock ]; do sleep 1; done
+while ! ctr list 2> /dev/null; do sleep 1; done
+
+# start system containers
+# temporarily using runc not containerd
+LOG=/var/log/system-containers.log
+touch $LOG
+for f in /containers/*
+do
+	base="$(basename $f)"
+	/sbin/start-stop-daemon --start --pidfile /run/$base.pid --exec /usr/bin/runc -- run --bundle "$f" --pid-file /run/$base.pid "$(basename $f)" </dev/null 2>$LOG >$LOG &
+	printf " - $(basename $f)\n"
+done
+
+wait

base/init/etc/init.d/rcS

@@ -0,0 +1,97 @@
+#!/bin/sh
+
+# mount filesystems
+mkdir -p -m 0755 /proc /run /tmp /sys /dev
+
+mount -n -t proc proc /proc -o ndodev,nosuid,noexec,relatime
+
+mount -n -t tmpfs tmpfs /run -o nodev,nosuid,noexec,relatime,size=10%,mode=755
+mount -n -t tmpfs tmpfs /tmp -o nodev,nosuid,noexec,relatime,size=10%,mode=1777
+
+# mount devfs
+mount -n -t devtmpfs dev /dev -o nosuid,noexec,relatime,size=10m,nr_inodes=248418,mode=755
+# devices
+[ -c /dev/console ] || mknod -m 600 /dev/console c 5 1
+[ -c /dev/tty1 ] || mknod -m 620 /dev/tty1 c 4 1
+[ -c /dev/tty ] || mknod -m 666 /dev/tty c 5 0
+
+[ -c /dev/null ] || mknod -m 666 /dev/null c 1 3
+[ -c /dev/kmsg ] || mknod -m 660 /dev/kmsg c 1 11
+
+# extra symbolic links not provided by default
+[ -e /dev/fd ] || ln -snf /proc/self/fd /dev/fd
+[ -e /dev/stdin ] || ln -snf /proc/self/fd/0 /dev/stdin
+[ -e /dev/stdout ] || ln -snf /proc/self/fd/1 /dev/stdout
+[ -e /dev/stderr ] || ln -snf /proc/self/fd/2 /dev/stderr
+[ -e /proc/kcore ] && ln -snf /proc/kcore /dev/core
+
+# devfs filesystems
+mkdir -p -m 1777 /dev/mqueue
+mkdir -p -m 1777 /dev/shm
+mkdir -p -m 0755 /dev/pts
+mount -n -t mqueue -o noexec,nosuid,nodev mqueue /dev/mqueue
+mount -n -t tmpfs -o noexec,nosuid,nodev,mode=1777 shm /dev/shm
+mount -n -t devpts -o noexec,nosuid,gid=5,mode=0620 devpts /dev/pts
+
+# mount sysfs
+sysfs_opts=nodev,noexec,nosuid
+mount -n -t sysfs -o ${sysfs_opts} sysfs /sys
+[ -d /sys/kernel/security ] && mount -n -t securityfs -o ${sysfs_opts} securityfs /sys/kernel/security
+[ -d /sys/kernel/debug ] && mount -n -t debugfs -o ${sysfs_opts} debugfs /sys/kernel/debug
+[ -d /sys/kernel/config ] && mount -n -t configfs -o ${sysfs_opts} configfs /sys/kernel/config
+[ -d /sys/fs/fuse/connections ] && mount -n -t fusectl -o ${sysfs_opts} fusectl /sys/fs/fuse/connections
+[ -d /sys/fs/selinux ] && mount -n -t selinuxfs -o nosuid,noexec selinuxfs /sys/fs/selinux
+[ -d /sys/fs/pstore ] && mount -n -t pstore pstore -o ${sysfs_opts} /sys/fs/pstore
+[ -d /sys/firmware/efi/efivars ] && mount -n -t efivarfs -o ro,${sysfs_opts} efivarfs /sys/firmware/efi/efivars
+
+# misc /proc mounted fs
+[ -d /proc/sys/fs/binfmt_misc ] && mount -t binfmt_misc -o nodev,noexec,nosuid binfmt_misc /proc/sys/fs/binfmt_misc
+
+# mount cgroups
+mount -n -t tmpfs -o nodev,noexec,nosuid,mode=755,size=10m cgroup_root /sys/fs/cgroup
+
+while read name hier groups enabled rest
+do
+	case "${enabled}" in
+	1)	mkdir -p /sys/fs/cgroup/${name}
+		mount -n -t cgroup -o ${sysfs_opts},${name} ${name} /sys/fs/cgroup/${name}
+	;;
+	esac
+done < /proc/cgroups
+
+# for compatibility
+mkdir -p /sys/fs/cgroup/systemd
+mount -t cgroup -o none,name=systemd cgroup /sys/fs/cgroup/systemd
+
+# set SELinux contexts
+if [ -x /sbin/restorecon ]
+then
+	restorecon -F /sys/devices/system/cpu/online >/dev/null 2>&1
+	restorecon -rF /sys/fs/cgroup >/dev/null 2>&1
+	restorecon -rF /dev >/dev/null 2>&1
+fi
+
+# start mdev for hotplug
+echo "/sbin/mdev" > /proc/sys/kernel/hotplug
+
+# mdev -s will not create /dev/usb[1-9] devices with recent kernels
+# so we trigger hotplug events for usb for now
+for i in $(find /sys/devices -name 'usb[0-9]*'); do
+	[ -e $i/uevent ] && echo add > $i/uevent
+done
+
+mdev -s
+
+# set hostname
+[ -s /etc/hostname ] && hostname -F /etc/hostname
+
+# set system clock from hwclock
+hwclock --hctosys --utc
+
+# bring up loopback interface
+ip addr add 127.0.0.1/8 dev lo brd + scope host
+ip route add 127.0.0.0/8 dev lo scope host
+ip link set lo up
+
+# will be containerised
+/sbin/dhcpcd

base/init/etc/inittab

@@ -0,0 +1,11 @@
+# /etc/inittab
+
+::sysinit:/etc/init.d/rcS
+::once:/etc/init.d/containerd
+::once:/etc/init.d/containers
+
+# Stuff to do for the 3-finger salute
+::ctrlaltdel:/sbin/reboot
+
+# Stuff to do before rebooting
+#::shutdown:/sbin/openrc shutdown

base/init/etc/issue

@@ -0,0 +1,12 @@
+
+Welcome to Moby
+
+                        ##         .
+                  ## ## ##        ==
+               ## ## ## ## ##    ===
+           /"""""""""""""""""\___/ ===
+      ~~~ {~~ ~~~~ ~~~ ~~~~ ~~~ ~ /  ===- ~~~
+           \______ o           __/
+             \    \         __/
+              \____\_______/
+

base/init/init

@@ -0,0 +1,44 @@
+#!/bin/sh
+
+setup_console() {
+	tty=${1%,*}
+	speed=${1#*,}
+	inittab="$2"
+	securetty="$3"
+	line=
+	term="linux"
+	[ "$speed" = "$1" ] && speed=115200
+
+	case "$tty" in
+	ttyS*|ttyAMA*|ttyUSB*|ttyMFD*)
+		line="-L"
+		term="vt100"
+		;;
+	tty0)
+		# skip current console
+		return 0
+		;;
+	esac
+	# skip consoles already in inittab
+	grep -q "^$tty:" "$inittab" && return
+
+	echo "$tty::once:cat /etc/issue" >> "$inittab"
+	echo "$tty::respawn:/sbin/getty -n -l /bin/sh $line $speed $tty $term" >> "$inittab"
+	if ! grep -q -w "$tty" "$securetty"; then
+		echo "$tty" >> "$securetty"
+	fi
+}
+
+/bin/mount -t tmpfs tmpfs /mnt
+
+/bin/cp -a / /mnt 2>/dev/null
+
+/bin/mount -t proc -o noexec,nosuid,nodev proc /proc
+for opt in $(cat /proc/cmdline); do
+	case "$opt" in
+	console=*)
+		setup_console ${opt#console=} /mnt/etc/inittab /mnt/etc/securetty;;
+	esac
+done
+
+exec /bin/busybox switch_root /mnt /sbin/init

base/runc/Dockerfile

@@ -10,7 +10,7 @@ RUN \
   linux-headers \
   make \
   && true
-ENV RUNC_COMMIT=51371867a01c467f08af739783b8beafc154c4d7
+ENV RUNC_COMMIT=ce450bcc6c135cae93ee2a99d41a308c179ff6dc
 RUN mkdir -p $GOPATH/src/github.com/opencontainers && \
   cd $GOPATH/src/github.com/opencontainers && \
   git clone https://github.com/opencontainers/runc.git

tools/c-compile/Dockerfile

@@ -2,6 +2,7 @@ FROM alpine:3.5
 RUN \
   apk update && apk upgrade && \
   apk add \
+  bsd-compat-headers \
   curl \
   gcc \
   git \

tools/docker2tar/Dockerfile

@@ -0,0 +1,5 @@
+FROM docker:1.13.1
+
+COPY . ./
+
+ENTRYPOINT ["/docker2tar.sh"]

tools/docker2tar/Makefile

@@ -0,0 +1,29 @@
+.PHONY: tag push
+
+BASE=docker:1.13.1
+IMAGE=docker2tar
+
+default: push
+
+hash: Dockerfile docker2tar.sh
+	DOCKER_CONTENT_TRUST=1 docker pull $(BASE)
+	tar cf - $^ | docker build --no-cache -t $(IMAGE):build -
+	docker run --entrypoint=/bin/sh --rm $(IMAGE):build -c 'cat $^ /usr/local/bin/* /lib/apk/db/installed | sha1sum' | sed 's/ .*//' > $@
+
+push: hash
+	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+		 (docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash) && \
+		  docker push mobylinux/$(IMAGE):$(shell cat hash))
+	docker rmi $(IMAGE):build
+	rm -f hash
+
+tag: hash
+	docker pull mobylinux/$(IMAGE):$(shell cat hash) || \
+		docker tag $(IMAGE):build mobylinux/$(IMAGE):$(shell cat hash)
+	docker rmi $(IMAGE):build
+	rm -f hash
+
+clean:
+	rm -f hash
+
+.DELETE_ON_ERROR:

tools/docker2tar/docker2tar.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+
+set -e
+
+# arguments is image name
+
+IMAGE="$1"; shift
+
+cd /tmp
+
+# extract rootfs
+EXCLUDE="--exclude .dockerenv --exclude Dockerfile \
+        --exclude dev/console --exclude dev/pts --exclude dev/shm \
+        --exclude etc/hostname --exclude etc/hosts --exclude etc/mtab --exclude etc/resolv.conf"
+CONTAINER="$(docker create $IMAGE /dev/null)"
+docker export "$CONTAINER" | tar -xf - $EXCLUDE
+docker rm "$CONTAINER" > /dev/null
+
+# these three files are bind mounted in by docker so they are not what we want
+
+mkdir -p etc
+
+cat << EOF > etc/hosts
+127.0.0.1       localhost
+::1     localhost ip6-localhost ip6-loopback
+fe00::0 ip6-localnet
+ff00::0 ip6-mcastprefix
+ff02::1 ip6-allnodes
+ff02::2 ip6-allrouters
+EOF
+
+cat << EOF > etc/resolv.conf
+nameserver 8.8.8.8
+nameserver 8.8.4.4
+nameserver 2001:4860:4860::8888
+nameserver 2001:4860:4860::8844
+EOF
+
+printf 'moby' > etc/hostname
+
+ln -s /proc/mounts etc/mtab
+
+tar cf - .