Make service start up containerd and services

This moves most of the initialisation of containers to the
service init in the `service` command.

Still leaves remounting root file systems read only but this
will go away shortly. Another step closer to removing shell
scripts in base system.

Signed-off-by: Justin Cormack <justin.cormack@docker.com>

pkg/containerd/cmd/service/main.go

@@ -9,6 +9,12 @@ import (
 	log "github.com/Sirupsen/logrus"
 )
 
+const (
+	defaultSocket     = "/run/containerd/containerd.sock"
+	defaultPath       = "/containers/services"
+	defaultContainerd = "/usr/bin/containerd"
+)
+
 var (
 	defaultLogFormatter = &log.TextFormatter{}
 )
@@ -67,9 +73,9 @@ func main() {
 
 	switch args[0] {
 	case "start":
-		start(args[1:])
+		startCmd(args[1:])
 	case "system-init":
-		systemInit(args[1:])
+		systemInitCmd(args[1:])
 	default:
 		fmt.Printf("%q is not valid command.\n\n", args[0])
 		flag.Usage()

pkg/containerd/cmd/service/start.go

@@ -15,7 +15,7 @@ import (
 	specs "github.com/opencontainers/runtime-spec/specs-go"
 )
 
-func start(args []string) {
+func startCmd(args []string) {
 	invoked := filepath.Base(os.Args[0])
 	flags := flag.NewFlagSet("start", flag.ExitOnError)
 	flags.Usage = func() {
@@ -24,7 +24,8 @@ func start(args []string) {
 		flags.PrintDefaults()
 	}
 
-	sock := flags.String("sock", "/run/containerd/containerd.sock", "Path to containerd socket")
+	sock := flags.String("sock", defaultSocket, "Path to containerd socket")
+	path := flags.String("path", defaultPath, "Path to service configs")
 
 	dumpSpec := flags.String("dump-spec", "", "Dump container spec to file before start")
 
@@ -40,55 +41,65 @@ func start(args []string) {
 	}
 
 	service := args[0]
-	rootfs := filepath.Join("/containers/services", service, "rootfs")
+
 	log.Infof("Starting service: %q", service)
 	log := log.WithFields(log.Fields{
 		"service": service,
 	})
 
-	client, err := containerd.New(*sock)
+	id, pid, msg, err := start(service, *sock, *path, *dumpSpec)
 	if err != nil {
-		log.WithError(err).Fatal("creating containerd client")
+		log.WithError(err).Fatal(msg)
+	}
+
+	log.Debugf("Started %s pid %d", id, pid)
+}
+
+func start(service, sock, path, dumpSpec string) (string, uint32, string, error) {
+	rootfs := filepath.Join(path, service, "rootfs")
+
+	client, err := containerd.New(sock)
+	if err != nil {
+		return "", 0, "creating containerd client", err
 	}
 
 	ctx := namespaces.WithNamespace(context.Background(), "default")
 
 	var spec *specs.Spec
-	specf, err := os.Open(filepath.Join("/containers/services", service, "config.json"))
+	specf, err := os.Open(filepath.Join(path, service, "config.json"))
 	if err != nil {
-		log.WithError(err).Fatal("failed to read service spec")
+		return "", 0, "failed to read service spec", err
 	}
 	if err := json.NewDecoder(specf).Decode(&spec); err != nil {
-		log.WithError(err).Fatal("failed to parse service spec")
+		return "", 0, "failed to parse service spec", err
 	}
 
-	log.Debugf("Rootfs is %s", rootfs)
-
 	spec.Root.Path = rootfs
 
-	if *dumpSpec != "" {
+	if dumpSpec != "" {
-		d, err := os.Create(*dumpSpec)
+		d, err := os.Create(dumpSpec)
 		if err != nil {
-			log.WithError(err).Fatal("failed to open file for spec dump")
+			return "", 0, "failed to open file for spec dump", err
 		}
 		enc := json.NewEncoder(d)
 		enc.SetIndent("", "    ")
 		if err := enc.Encode(&spec); err != nil {
-			log.WithError(err).Fatal("failed to write spec dump")
+			return "", 0, "failed to write spec dump", err
 		}
 
 	}
 
 	ctr, err := client.NewContainer(ctx, service, containerd.WithSpec(spec))
 	if err != nil {
-		log.WithError(err).Fatal("failed to create container")
+		return "", 0, "failed to create container", err
 	}
 
 	io := func() (*containerd.IO, error) {
 		logfile := filepath.Join("/var/log", service+".log")
 		// We just need this to exist.
-		if err := ioutil.WriteFile(logfile, []byte{}, 0666); err != nil {
+		if err := ioutil.WriteFile(logfile, []byte{}, 0600); err != nil {
-			log.WithError(err).Fatal("failed to touch logfile")
+			// if we cannot write to log, discard output
+			logfile = "/dev/null"
 		}
 		return &containerd.IO{
 			Stdin:    "/dev/null",
@@ -101,13 +112,13 @@ func start(args []string) {
 	task, err := ctr.NewTask(ctx, io)
 	if err != nil {
 		// Don't bother to destroy the container here.
-		log.WithError(err).Fatal("failed to create task")
+		return "", 0, "failed to create task", err
 	}
 
 	if err := task.Start(ctx); err != nil {
 		// Don't destroy the container here so it can be inspected for debugging.
-		log.WithError(err).Fatal("failed to start task")
+		return "", 0, "failed to start task", err
 	}
 
-	log.Debugf("Started %s pid %d", ctr.ID(), task.Pid())
+	return ctr.ID(), task.Pid(), "", nil
 }

pkg/containerd/cmd/service/system_init.go

@@ -4,9 +4,12 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"syscall"
+	"time"
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/containerd/containerd"
@@ -49,7 +52,7 @@ func cleanupTask(ctx context.Context, ctr containerd.Container) error {
 	}
 }
 
-func systemInit(args []string) {
+func systemInitCmd(args []string) {
 	invoked := filepath.Base(os.Args[0])
 	flags := flag.NewFlagSet("system-init", flag.ExitOnError)
 	flags.Usage = func() {
@@ -58,7 +61,9 @@ func systemInit(args []string) {
 		flags.PrintDefaults()
 	}
 
-	sock := flags.String("sock", "/run/containerd/containerd.sock", "Path to containerd socket")
+	sock := flags.String("sock", defaultSocket, "Path to containerd socket")
+	path := flags.String("path", defaultPath, "Path to service configs")
+	binary := flags.String("containerd", defaultContainerd, "Path to containerd")
 
 	if err := flags.Parse(args); err != nil {
 		log.Fatal("Unable to parse args")
@@ -71,6 +76,33 @@ func systemInit(args []string) {
 		os.Exit(1)
 	}
 
+	// remove (unlikely) old containerd socket
+	_ = os.Remove(*sock)
+
+	// start up containerd
+	cmd := exec.Command(*binary)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		log.WithError(err).Fatal("cannot start containerd")
+	}
+
+	// wait for containerd socket to appear
+	for {
+		_, err := os.Stat(*sock)
+		if err == nil {
+			break
+		}
+		err = cmd.Process.Signal(syscall.Signal(0))
+		if err != nil {
+			// process not there, wait() to find error
+			err = cmd.Wait()
+			log.WithError(err).Fatal("containerd process exited")
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// connect to containerd
 	client, err := containerd.New(*sock)
 	if err != nil {
 		log.WithError(err).Fatal("creating containerd client")
@@ -83,6 +115,7 @@ func systemInit(args []string) {
 		log.WithError(err).Fatal("listing containers")
 	}
 
+	// Clean up any old containers
 	// None of the errors in this loop are fatal since we want to
 	// keep trying.
 	for _, ctr := range ctrs {
@@ -99,4 +132,18 @@ func systemInit(args []string) {
 			log.WithError(err).Error("deleting container")
 		}
 	}
+
+	// Start up containers
+	files, err := ioutil.ReadDir(*path)
+	// just skip if there is an error, eg no such path
+	if err != nil {
+		return
+	}
+	for _, file := range files {
+		if id, pid, msg, err := start(file.Name(), *sock, *path, ""); err != nil {
+			log.WithError(err).Error(msg)
+		} else {
+			log.Debugf("Started %s pid %d", id, pid)
+		}
+	}
 }

pkg/containerd/etc/init.d/020-containerd

@@ -1,25 +1,14 @@
 #!/bin/sh
 
-# bring up containerd
-printf "\nStarting containerd\n"
-/usr/bin/containerd &
-
-# wait for socket to be there
-while [ ! -S /run/containerd/containerd.sock ]
-do
-	sleep 0.1
-done
-
 # start service containers
 
-service system-init
-
 if [ -d /containers/services ]
 then
 	for f in $(find /containers/services -mindepth 1 -maxdepth 1 | sort)
 	do
 		/bin/mount --bind "$f/rootfs" "$f/rootfs"
 		mount -o remount,rw "$f/rootfs"
-		service start "$(basename $f)"
 	done
 fi
+
+service system-init