Add a runtime config

This adds support for a runtime configuration file that can do:
- `mkdir` to make a directory at runtime, eg in `/var` or `/tmp`, to avoid workarounds
- `interface` that can create network interfaces in a container or move them
- `bindNS` that can bind mount namespaces of an `onboot` container to a file so a service can be started in that namespace.

It merges the `service` and `onboot` tools (in `init`) to avoid duplication. This also saves some size for
eg LCOW which did not use the `onboot` code in `runc`.

Signed-off-by: Justin Cormack <justin.cormack@docker.com>

pkg/init/cmd/service/main.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultSocket     = "/run/containerd/containerd.sock"
+	defaultPath       = "/containers/services"
+	defaultContainerd = "/usr/bin/containerd"
+	installPath       = "/usr/bin/service"
+	onbootPath        = "/containers/onboot"
+	shutdownPath      = "/containers/onshutdown"
+)
+
+var (
+	defaultLogFormatter = &log.TextFormatter{}
+)
+
+// infoFormatter overrides the default format for Info() log events to
+// provide an easier to read output
+type infoFormatter struct {
+}
+
+func (f *infoFormatter) Format(entry *log.Entry) ([]byte, error) {
+	if entry.Level == log.InfoLevel {
+		return append([]byte(entry.Message), '\n'), nil
+	}
+	return defaultLogFormatter.Format(entry)
+}
+
+func main() {
+	flag.Usage = func() {
+		fmt.Printf("USAGE: %s [options] COMMAND\n\n", filepath.Base(os.Args[0]))
+		fmt.Printf("Commands:\n")
+		fmt.Printf("  system-init Prepare the system at start of day\n")
+		fmt.Printf("  start       Start a service\n")
+		fmt.Printf("  help        Print this message\n")
+		fmt.Printf("\n")
+		fmt.Printf("Run '%s COMMAND --help' for more information on the command\n", filepath.Base(os.Args[0]))
+		fmt.Printf("\n")
+		fmt.Printf("Options:\n")
+		flag.PrintDefaults()
+	}
+	flagQuiet := flag.Bool("q", false, "Quiet execution")
+	flagVerbose := flag.Bool("v", false, "Verbose execution")
+
+	// Set up logging
+	log.SetFormatter(new(infoFormatter))
+	log.SetLevel(log.InfoLevel)
+	flag.Parse()
+	if *flagQuiet && *flagVerbose {
+		fmt.Printf("Can't set quiet and verbose flag at the same time\n")
+		os.Exit(1)
+	}
+	if *flagQuiet {
+		log.SetLevel(log.ErrorLevel)
+	}
+	if *flagVerbose {
+		// Switch back to the standard formatter
+		log.SetFormatter(defaultLogFormatter)
+		log.SetLevel(log.DebugLevel)
+	}
+
+	args := flag.Args()
+	if len(args) < 1 {
+		// check if called form startup scripts
+		command := os.Args[0]
+		switch {
+		case strings.Contains(command, "onboot"):
+			os.Exit(runcInit(onbootPath))
+		case strings.Contains(command, "onshutdown"):
+			os.Exit(runcInit(shutdownPath))
+		case strings.Contains(command, "containerd"):
+			systemInitCmd([]string{})
+			os.Exit(0)
+		}
+	}
+
+	switch args[0] {
+	case "start":
+		startCmd(args[1:])
+	case "system-init":
+		systemInitCmd(args[1:])
+	default:
+		fmt.Printf("%q is not valid command.\n\n", args[0])
+		flag.Usage()
+		os.Exit(1)
+	}
+}

pkg/init/cmd/service/prepare.go

@@ -0,0 +1,255 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"syscall"
+
+	"github.com/vishvananda/netlink"
+)
+
+const (
+	wgPath      = "/usr/bin/wg"
+	nsenterPath = "/usr/bin/nsenter-net"
+)
+
+// Note these definitions are from moby/tool/src/moby/config.go and should be kept in sync
+
+// Runtime is the type of config processed at runtime, not used to build the OCI spec
+type Runtime struct {
+	Mkdir      []string    `yaml:"mkdir" json:"mkdir,omitempty"`
+	Interfaces []Interface `yaml:"interfaces" json:"interfaces,omitempty"`
+	BindNS     *Namespaces `yaml:"bindNS" json:"bindNS,omitempty"`
+}
+
+// Namespaces is the type for configuring paths to bind namespaces
+type Namespaces struct {
+	Cgroup string `yaml:"cgroup" json:"cgroup,omitempty"`
+	Ipc    string `yaml:"ipc" json:"ipc,omitempty"`
+	Mnt    string `yaml:"mnt" json:"mnt,omitempty"`
+	Net    string `yaml:"net" json:"net,omitempty"`
+	Pid    string `yaml:"pid" json:"pid,omitempty"`
+	User   string `yaml:"user" json:"user,omitempty"`
+	Uts    string `yaml:"uts" json:"uts,omitempty"`
+}
+
+// Interface is the runtime config for network interfaces
+type Interface struct {
+	Name         string `yaml:"name" json:"name,omitempty"`
+	Add          string `yaml:"add" json:"add,omitempty"`
+	Peer         string `yaml:"peer" json:"peer,omitempty"`
+	CreateInRoot bool   `yaml:"createInRoot" json:"createInRoot"`
+}
+
+func getRuntimeConfig(path string) Runtime {
+	var runtime Runtime
+	conf, err := ioutil.ReadFile(filepath.Join(path, "runtime.json"))
+	if err != nil {
+		// if it does not exist it is fine to return an empty runtime, to not do anything
+		if os.IsNotExist(err) {
+			return runtime
+		}
+		log.Fatalf("Cannot read runtime config: %v", err)
+	}
+	if err := json.Unmarshal(conf, &runtime); err != nil {
+		log.Fatalf("Cannot parse runtime config: %v", err)
+	}
+	return runtime
+}
+
+// prepareFilesystem sets up the mounts, before the container is created
+func prepareFilesystem(path string, runtime Runtime) error {
+	// execute the runtime config that should be done up front
+	for _, dir := range runtime.Mkdir {
+		// in future we may need to change the structure to set mode, ownership
+		var mode os.FileMode = 0755
+		err := os.MkdirAll(dir, mode)
+		if err != nil {
+			return fmt.Errorf("Cannot create directory %s: %v", dir, err)
+		}
+	}
+
+	// see if we are dealing with a read only or read write container
+	if _, err := os.Stat(filepath.Join(path, "lower")); err != nil {
+		if os.IsNotExist(err) {
+			return prepareRO(path)
+		}
+		return err
+	}
+	return prepareRW(path)
+}
+
+func prepareRO(path string) error {
+	// make rootfs a mount point, as runc doesn't like it much otherwise
+	rootfs := filepath.Join(path, "rootfs")
+	if err := syscall.Mount(rootfs, rootfs, "", syscall.MS_BIND, ""); err != nil {
+		return err
+	}
+	return nil
+}
+
+func prepareRW(path string) error {
+	// mount a tmpfs on tmp for upper and workdirs
+	// make it private as nothing else should be using this
+	tmp := filepath.Join(path, "tmp")
+	if err := syscall.Mount("tmpfs", tmp, "tmpfs", 0, "size=10%"); err != nil {
+		return err
+	}
+	// make it private as nothing else should be using this
+	if err := syscall.Mount("", tmp, "", syscall.MS_REMOUNT|syscall.MS_PRIVATE, ""); err != nil {
+		return err
+	}
+	upper := filepath.Join(tmp, "upper")
+	// make the mount points
+	if err := os.Mkdir(upper, 0755); err != nil {
+		return err
+	}
+	work := filepath.Join(tmp, "work")
+	if err := os.Mkdir(work, 0755); err != nil {
+		return err
+	}
+	lower := filepath.Join(path, "lower")
+	rootfs := filepath.Join(path, "rootfs")
+	opt := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lower, upper, work)
+	if err := syscall.Mount("overlay", rootfs, "overlay", 0, opt); err != nil {
+		return err
+	}
+	return nil
+}
+
+// bind mount a namespace file
+func bindNS(ns string, path string, pid int) error {
+	if path == "" {
+		return nil
+	}
+	// the path and file need to exist for the bind to succeed, so try to create
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return fmt.Errorf("Cannot create leading directories %s for bind mount destination: %v", dir, err)
+	}
+	fi, err := os.Create(path)
+	if err != nil {
+		return fmt.Errorf("Cannot create a mount point for namespace bind at %s: %v", path, err)
+	}
+	if err := fi.Close(); err != nil {
+		return err
+	}
+	if err := syscall.Mount(fmt.Sprintf("/proc/%d/ns/%s", pid, ns), path, "", syscall.MS_BIND, ""); err != nil {
+		return fmt.Errorf("Failed to bind %s namespace at %s: %v", ns, path, err)
+	}
+	return nil
+}
+
+// prepareProcess sets up anything that needs to be done after the container process is created, but before it runs
+// for example networking
+func prepareProcess(pid int, runtime Runtime) error {
+	for _, iface := range runtime.Interfaces {
+		if iface.Name == "" {
+			return fmt.Errorf("Interface requires a name")
+		}
+
+		var link netlink.Link
+		var ns interface{} = netlink.NsPid(pid)
+		var move bool
+		var err error
+
+		if iface.Peer != "" && iface.Add == "" {
+			// must be a veth if specify peer
+			iface.Add = "veth"
+		}
+
+		// if create in root is set, create in root namespace first, then move
+		// also do the same for a veth pair
+		if iface.CreateInRoot || iface.Add == "veth" {
+			ns = nil
+			move = true
+		}
+
+		if iface.Add != "" {
+			switch iface.Add {
+			case "veth":
+				if iface.Peer == "" {
+					return fmt.Errorf("Creating a veth pair %s requires a peer to be set", iface.Name)
+				}
+				la := netlink.LinkAttrs{Name: iface.Name, Namespace: ns}
+				link = &netlink.Veth{LinkAttrs: la, PeerName: iface.Peer}
+			default:
+				// no special creation options needed
+				la := netlink.LinkAttrs{Name: iface.Name, Namespace: ns}
+				link = &netlink.GenericLink{la, iface.Add}
+			}
+			if err := netlink.LinkAdd(link); err != nil {
+				return fmt.Errorf("Link add %s of type %s failed: %v", iface.Name, iface.Add, err)
+			}
+			fmt.Fprintf(os.Stderr, "Created interface %s type %s\n", iface.Name, iface.Add)
+		} else {
+			// find existing interface
+			link, err = netlink.LinkByName(iface.Name)
+			if err != nil {
+				return fmt.Errorf("Cannot find interface %s: %v", iface.Name, err)
+			}
+			// then move into namespace
+			move = true
+		}
+		if move {
+			if err := netlink.LinkSetNsPid(link, int(pid)); err != nil {
+				return fmt.Errorf("Cannot move interface %s into namespace: %v", iface.Name, err)
+			}
+			fmt.Fprintf(os.Stderr, "Moved interface %s to pid %d\n", iface.Name, pid)
+		}
+	}
+
+	if runtime.BindNS != nil {
+		binds := []struct {
+			ns   string
+			path string
+		}{
+			{"cgroup", runtime.BindNS.Cgroup},
+			{"ipc", runtime.BindNS.Ipc},
+			{"mnt", runtime.BindNS.Mnt},
+			{"net", runtime.BindNS.Net},
+			{"pid", runtime.BindNS.Pid},
+			{"user", runtime.BindNS.User},
+			{"uts", runtime.BindNS.Uts},
+		}
+
+		for _, b := range binds {
+			if err := bindNS(b.ns, b.path, pid); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+// cleanup functions are best efforts only, mainly for rw onboot containers
+func cleanup(path string) {
+	// see if we are dealing with a read only or read write container
+	if _, err := os.Stat(filepath.Join(path, "lower")); err != nil {
+		cleanupRO(path)
+	} else {
+		cleanupRW(path)
+	}
+}
+
+func cleanupRO(path string) {
+	// remove the bind mount
+	rootfs := filepath.Join(path, "rootfs")
+	_ = syscall.Unmount(rootfs, 0)
+}
+
+func cleanupRW(path string) {
+	// remove the overlay mount
+	rootfs := filepath.Join(path, "rootfs")
+	_ = os.RemoveAll(rootfs)
+	_ = syscall.Unmount(rootfs, 0)
+	// remove the tmpfs
+	tmp := filepath.Join(path, "tmp")
+	_ = os.RemoveAll(tmp)
+	_ = syscall.Unmount(tmp, 0)
+}

pkg/init/cmd/service/runc.go

@@ -0,0 +1,111 @@
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+
+	"github.com/containerd/containerd/sys"
+)
+
+const (
+	runcBinary = "/usr/bin/runc"
+)
+
+func runcInit(rootPath string) int {
+	// do nothing if the path does not exist
+	if _, err := os.Stat(rootPath); err != nil && os.IsNotExist(err) {
+		return 0
+	}
+
+	// get files; note ReadDir already sorts them
+	files, err := ioutil.ReadDir(rootPath)
+	if err != nil {
+		log.Fatalf("Cannot read files in %s: %v", rootPath, err)
+	}
+
+	tmpdir, err := ioutil.TempDir("", filepath.Base(rootPath))
+	if err != nil {
+		log.Fatalf("Cannot create temporary directory: %v", err)
+	}
+
+	// need to set ourselves as a child subreaper or we cannot wait for runc as reparents to init
+	if err := sys.SetSubreaper(1); err != nil {
+		log.Fatalf("Cannot set as subreaper: %v", err)
+	}
+
+	status := 0
+
+	for _, file := range files {
+		name := file.Name()
+		path := filepath.Join(rootPath, name)
+
+		runtimeConfig := getRuntimeConfig(path)
+
+		if err := prepareFilesystem(path, runtimeConfig); err != nil {
+			log.Printf("Error preparing %s: %v", name, err)
+			status = 1
+			continue
+		}
+		pidfile := filepath.Join(tmpdir, name)
+		cmd := exec.Command(runcBinary, "create", "--bundle", path, "--pid-file", pidfile, name)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			log.Printf("Error creating %s: %v", name, err)
+			status = 1
+			// skip cleanup on error for debug
+			continue
+		}
+		pf, err := ioutil.ReadFile(pidfile)
+		if err != nil {
+			log.Printf("Cannot read pidfile: %v", err)
+			status = 1
+			continue
+		}
+		pid, err := strconv.Atoi(string(pf))
+		if err != nil {
+			log.Printf("Cannot parse pid from pidfile: %v", err)
+			status = 1
+			continue
+		}
+
+		if err := prepareProcess(pid, runtimeConfig); err != nil {
+			log.Printf("Cannot prepare process: %v", err)
+			status = 1
+			continue
+		}
+
+		waitFor := make(chan *os.ProcessState)
+		go func() {
+			// never errors in Unix
+			p, _ := os.FindProcess(pid)
+			state, err := p.Wait()
+			if err != nil {
+				log.Printf("Process wait error: %v", err)
+			}
+			waitFor <- state
+		}()
+
+		cmd = exec.Command(runcBinary, "start", name)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			log.Printf("Error starting %s: %v", name, err)
+			status = 1
+			continue
+		}
+
+		_ = <-waitFor
+
+		cleanup(path)
+		_ = os.Remove(pidfile)
+	}
+
+	_ = os.RemoveAll(tmpdir)
+
+	return status
+}

pkg/init/cmd/service/start.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/namespaces"
+	specs "github.com/opencontainers/runtime-spec/specs-go"
+	log "github.com/sirupsen/logrus"
+)
+
+func startCmd(args []string) {
+	invoked := filepath.Base(os.Args[0])
+	flags := flag.NewFlagSet("start", flag.ExitOnError)
+	flags.Usage = func() {
+		fmt.Printf("USAGE: %s start [service]\n\n", invoked)
+		fmt.Printf("Options:\n")
+		flags.PrintDefaults()
+	}
+
+	sock := flags.String("sock", defaultSocket, "Path to containerd socket")
+	path := flags.String("path", defaultPath, "Path to service configs")
+
+	dumpSpec := flags.String("dump-spec", "", "Dump container spec to file before start")
+
+	if err := flags.Parse(args); err != nil {
+		log.Fatal("Unable to parse args")
+	}
+	args = flags.Args()
+
+	if len(args) != 1 {
+		fmt.Println("Please specify the service")
+		flags.Usage()
+		os.Exit(1)
+	}
+
+	service := args[0]
+
+	log.Infof("Starting service: %q", service)
+	log := log.WithFields(log.Fields{
+		"service": service,
+	})
+
+	id, pid, msg, err := start(service, *sock, *path, *dumpSpec)
+	if err != nil {
+		log.WithError(err).Fatal(msg)
+	}
+
+	log.Debugf("Started %s pid %d", id, pid)
+}
+
+func start(service, sock, basePath, dumpSpec string) (string, uint32, string, error) {
+	path := filepath.Join(basePath, service)
+
+	runtimeConfig := getRuntimeConfig(path)
+
+	rootfs := filepath.Join(path, "rootfs")
+
+	if err := prepareFilesystem(path, runtimeConfig); err != nil {
+		return "", 0, "preparing filesystem", err
+	}
+
+	client, err := containerd.New(sock)
+	if err != nil {
+		return "", 0, "creating containerd client", err
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "default")
+
+	var spec *specs.Spec
+	specf, err := os.Open(filepath.Join(path, "config.json"))
+	if err != nil {
+		return "", 0, "failed to read service spec", err
+	}
+	if err := json.NewDecoder(specf).Decode(&spec); err != nil {
+		return "", 0, "failed to parse service spec", err
+	}
+
+	spec.Root.Path = rootfs
+
+	if dumpSpec != "" {
+		d, err := os.Create(dumpSpec)
+		if err != nil {
+			return "", 0, "failed to open file for spec dump", err
+		}
+		enc := json.NewEncoder(d)
+		enc.SetIndent("", "    ")
+		if err := enc.Encode(&spec); err != nil {
+			return "", 0, "failed to write spec dump", err
+		}
+
+	}
+
+	ctr, err := client.NewContainer(ctx, service, containerd.WithSpec(spec))
+	if err != nil {
+		return "", 0, "failed to create container", err
+	}
+
+	io := func(id string) (*containerd.IO, error) {
+		logfile := filepath.Join("/var/log", service+".log")
+		// We just need this to exist.
+		if err := ioutil.WriteFile(logfile, []byte{}, 0600); err != nil {
+			// if we cannot write to log, discard output
+			logfile = "/dev/null"
+		}
+		return &containerd.IO{
+			Stdin:    "/dev/null",
+			Stdout:   logfile,
+			Stderr:   logfile,
+			Terminal: false,
+		}, nil
+	}
+
+	task, err := ctr.NewTask(ctx, io)
+	if err != nil {
+		// Don't bother to destroy the container here.
+		return "", 0, "failed to create task", err
+	}
+
+	if err := prepareProcess(int(task.Pid()), runtimeConfig); err != nil {
+		return "", 0, "preparing process", err
+	}
+
+	if err := task.Start(ctx); err != nil {
+		// Don't destroy the container here so it can be inspected for debugging.
+		return "", 0, "failed to start task", err
+	}
+
+	return ctr.ID(), task.Pid(), "", nil
+}

pkg/init/cmd/service/system_init.go

@@ -0,0 +1,149 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd"
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+func cleanupTask(ctx context.Context, ctr containerd.Container) error {
+	task, err := ctr.Task(ctx, nil)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrap(err, "getting task")
+	}
+
+	deleteErr := make(chan error, 1)
+	deleteCtx, deleteCancel := context.WithCancel(ctx)
+	defer deleteCancel()
+
+	go func(ctx context.Context, ch chan error) {
+		_, err := task.Delete(ctx)
+		if err != nil {
+			ch <- errors.Wrap(err, "killing task")
+		}
+		ch <- nil
+	}(deleteCtx, deleteErr)
+
+	sig := syscall.SIGKILL
+	if err := task.Kill(ctx, sig); err != nil && !errdefs.IsNotFound(err) {
+		return errors.Wrapf(err, "killing task with %q", sig)
+	}
+
+	select {
+	case err := <-deleteErr:
+		return err
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+func systemInitCmd(args []string) {
+	invoked := filepath.Base(os.Args[0])
+	flags := flag.NewFlagSet("system-init", flag.ExitOnError)
+	flags.Usage = func() {
+		fmt.Printf("USAGE: %s system-init\n\n", invoked)
+		fmt.Printf("Options:\n")
+		flags.PrintDefaults()
+	}
+
+	sock := flags.String("sock", defaultSocket, "Path to containerd socket")
+	path := flags.String("path", defaultPath, "Path to service configs")
+	binary := flags.String("containerd", defaultContainerd, "Path to containerd")
+
+	if err := flags.Parse(args); err != nil {
+		log.Fatal("Unable to parse args")
+	}
+	args = flags.Args()
+
+	if len(args) != 0 {
+		fmt.Println("Unexpected argument")
+		flags.Usage()
+		os.Exit(1)
+	}
+
+	// remove (unlikely) old containerd socket
+	_ = os.Remove(*sock)
+
+	// start up containerd
+	cmd := exec.Command(*binary)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		log.WithError(err).Fatal("cannot start containerd")
+	}
+
+	// wait for containerd socket to appear
+	for {
+		_, err := os.Stat(*sock)
+		if err == nil {
+			break
+		}
+		err = cmd.Process.Signal(syscall.Signal(0))
+		if err != nil {
+			// process not there, wait() to find error
+			err = cmd.Wait()
+			log.WithError(err).Fatal("containerd process exited")
+		}
+		time.Sleep(100 * time.Millisecond)
+	}
+
+	// connect to containerd
+	client, err := containerd.New(*sock)
+	if err != nil {
+		log.WithError(err).Fatal("creating containerd client")
+	}
+
+	ctx := namespaces.WithNamespace(context.Background(), "default")
+
+	ctrs, err := client.Containers(ctx)
+	if err != nil {
+		log.WithError(err).Fatal("listing containers")
+	}
+
+	// Clean up any old containers
+	// None of the errors in this loop are fatal since we want to
+	// keep trying.
+	for _, ctr := range ctrs {
+		log.Infof("Cleaning up stale service: %q", ctr.ID())
+		log := log.WithFields(log.Fields{
+			"service": ctr.ID(),
+		})
+
+		if err := cleanupTask(ctx, ctr); err != nil {
+			log.WithError(err).Error("cleaning up task")
+		}
+
+		if err := ctr.Delete(ctx); err != nil {
+			log.WithError(err).Error("deleting container")
+		}
+	}
+
+	// Start up containers
+	files, err := ioutil.ReadDir(*path)
+	// just skip if there is an error, eg no such path
+	if err != nil {
+		return
+	}
+	for _, file := range files {
+		if id, pid, msg, err := start(file.Name(), *sock, *path, ""); err != nil {
+			log.WithError(err).Error(msg)
+		} else {
+			log.Debugf("Started %s pid %d", id, pid)
+		}
+	}
+}