From e40db1459891e3edf303a12996488d7613571a17 Mon Sep 17 00:00:00 2001 From: Justin Cormack Date: Wed, 26 Jul 2017 11:21:14 +0100 Subject: [PATCH] Make service start up containerd and services This moves most of the initialisation of containers to the service init in the `service` command. Still leaves remounting root file systems read only but this will go away shortly. Another step closer to removing shell scripts in base system. Signed-off-by: Justin Cormack --- pkg/containerd/cmd/service/main.go | 10 ++++- pkg/containerd/cmd/service/start.go | 51 ++++++++++++++--------- pkg/containerd/cmd/service/system_init.go | 51 ++++++++++++++++++++++- pkg/containerd/etc/init.d/020-containerd | 15 +------ 4 files changed, 90 insertions(+), 37 deletions(-) diff --git a/pkg/containerd/cmd/service/main.go b/pkg/containerd/cmd/service/main.go index 801c71078..3d6957f5c 100644 --- a/pkg/containerd/cmd/service/main.go +++ b/pkg/containerd/cmd/service/main.go @@ -9,6 +9,12 @@ import ( log "github.com/Sirupsen/logrus" ) +const ( + defaultSocket = "/run/containerd/containerd.sock" + defaultPath = "/containers/services" + defaultContainerd = "/usr/bin/containerd" +) + var ( defaultLogFormatter = &log.TextFormatter{} ) @@ -67,9 +73,9 @@ func main() { switch args[0] { case "start": - start(args[1:]) + startCmd(args[1:]) case "system-init": - systemInit(args[1:]) + systemInitCmd(args[1:]) default: fmt.Printf("%q is not valid command.\n\n", args[0]) flag.Usage() diff --git a/pkg/containerd/cmd/service/start.go b/pkg/containerd/cmd/service/start.go index 82004df90..e34c969dd 100644 --- a/pkg/containerd/cmd/service/start.go +++ b/pkg/containerd/cmd/service/start.go @@ -15,7 +15,7 @@ import ( specs "github.com/opencontainers/runtime-spec/specs-go" ) -func start(args []string) { +func startCmd(args []string) { invoked := filepath.Base(os.Args[0]) flags := flag.NewFlagSet("start", flag.ExitOnError) flags.Usage = func() { @@ -24,7 +24,8 @@ func start(args []string) { flags.PrintDefaults() } - sock := flags.String("sock", "/run/containerd/containerd.sock", "Path to containerd socket") + sock := flags.String("sock", defaultSocket, "Path to containerd socket") + path := flags.String("path", defaultPath, "Path to service configs") dumpSpec := flags.String("dump-spec", "", "Dump container spec to file before start") @@ -40,55 +41,65 @@ func start(args []string) { } service := args[0] - rootfs := filepath.Join("/containers/services", service, "rootfs") + log.Infof("Starting service: %q", service) log := log.WithFields(log.Fields{ "service": service, }) - client, err := containerd.New(*sock) + id, pid, msg, err := start(service, *sock, *path, *dumpSpec) if err != nil { - log.WithError(err).Fatal("creating containerd client") + log.WithError(err).Fatal(msg) + } + + log.Debugf("Started %s pid %d", id, pid) +} + +func start(service, sock, path, dumpSpec string) (string, uint32, string, error) { + rootfs := filepath.Join(path, service, "rootfs") + + client, err := containerd.New(sock) + if err != nil { + return "", 0, "creating containerd client", err } ctx := namespaces.WithNamespace(context.Background(), "default") var spec *specs.Spec - specf, err := os.Open(filepath.Join("/containers/services", service, "config.json")) + specf, err := os.Open(filepath.Join(path, service, "config.json")) if err != nil { - log.WithError(err).Fatal("failed to read service spec") + return "", 0, "failed to read service spec", err } if err := json.NewDecoder(specf).Decode(&spec); err != nil { - log.WithError(err).Fatal("failed to parse service spec") + return "", 0, "failed to parse service spec", err } - log.Debugf("Rootfs is %s", rootfs) - spec.Root.Path = rootfs - if *dumpSpec != "" { - d, err := os.Create(*dumpSpec) + if dumpSpec != "" { + d, err := os.Create(dumpSpec) if err != nil { - log.WithError(err).Fatal("failed to open file for spec dump") + return "", 0, "failed to open file for spec dump", err } enc := json.NewEncoder(d) enc.SetIndent("", " ") if err := enc.Encode(&spec); err != nil { - log.WithError(err).Fatal("failed to write spec dump") + return "", 0, "failed to write spec dump", err } } ctr, err := client.NewContainer(ctx, service, containerd.WithSpec(spec)) if err != nil { - log.WithError(err).Fatal("failed to create container") + return "", 0, "failed to create container", err } io := func() (*containerd.IO, error) { logfile := filepath.Join("/var/log", service+".log") // We just need this to exist. - if err := ioutil.WriteFile(logfile, []byte{}, 0666); err != nil { - log.WithError(err).Fatal("failed to touch logfile") + if err := ioutil.WriteFile(logfile, []byte{}, 0600); err != nil { + // if we cannot write to log, discard output + logfile = "/dev/null" } return &containerd.IO{ Stdin: "/dev/null", @@ -101,13 +112,13 @@ func start(args []string) { task, err := ctr.NewTask(ctx, io) if err != nil { // Don't bother to destroy the container here. - log.WithError(err).Fatal("failed to create task") + return "", 0, "failed to create task", err } if err := task.Start(ctx); err != nil { // Don't destroy the container here so it can be inspected for debugging. - log.WithError(err).Fatal("failed to start task") + return "", 0, "failed to start task", err } - log.Debugf("Started %s pid %d", ctr.ID(), task.Pid()) + return ctr.ID(), task.Pid(), "", nil } diff --git a/pkg/containerd/cmd/service/system_init.go b/pkg/containerd/cmd/service/system_init.go index b412e707b..c450d5f40 100644 --- a/pkg/containerd/cmd/service/system_init.go +++ b/pkg/containerd/cmd/service/system_init.go @@ -4,9 +4,12 @@ import ( "context" "flag" "fmt" + "io/ioutil" "os" + "os/exec" "path/filepath" "syscall" + "time" log "github.com/Sirupsen/logrus" "github.com/containerd/containerd" @@ -49,7 +52,7 @@ func cleanupTask(ctx context.Context, ctr containerd.Container) error { } } -func systemInit(args []string) { +func systemInitCmd(args []string) { invoked := filepath.Base(os.Args[0]) flags := flag.NewFlagSet("system-init", flag.ExitOnError) flags.Usage = func() { @@ -58,7 +61,9 @@ func systemInit(args []string) { flags.PrintDefaults() } - sock := flags.String("sock", "/run/containerd/containerd.sock", "Path to containerd socket") + sock := flags.String("sock", defaultSocket, "Path to containerd socket") + path := flags.String("path", defaultPath, "Path to service configs") + binary := flags.String("containerd", defaultContainerd, "Path to containerd") if err := flags.Parse(args); err != nil { log.Fatal("Unable to parse args") @@ -71,6 +76,33 @@ func systemInit(args []string) { os.Exit(1) } + // remove (unlikely) old containerd socket + _ = os.Remove(*sock) + + // start up containerd + cmd := exec.Command(*binary) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Start(); err != nil { + log.WithError(err).Fatal("cannot start containerd") + } + + // wait for containerd socket to appear + for { + _, err := os.Stat(*sock) + if err == nil { + break + } + err = cmd.Process.Signal(syscall.Signal(0)) + if err != nil { + // process not there, wait() to find error + err = cmd.Wait() + log.WithError(err).Fatal("containerd process exited") + } + time.Sleep(100 * time.Millisecond) + } + + // connect to containerd client, err := containerd.New(*sock) if err != nil { log.WithError(err).Fatal("creating containerd client") @@ -83,6 +115,7 @@ func systemInit(args []string) { log.WithError(err).Fatal("listing containers") } + // Clean up any old containers // None of the errors in this loop are fatal since we want to // keep trying. for _, ctr := range ctrs { @@ -99,4 +132,18 @@ func systemInit(args []string) { log.WithError(err).Error("deleting container") } } + + // Start up containers + files, err := ioutil.ReadDir(*path) + // just skip if there is an error, eg no such path + if err != nil { + return + } + for _, file := range files { + if id, pid, msg, err := start(file.Name(), *sock, *path, ""); err != nil { + log.WithError(err).Error(msg) + } else { + log.Debugf("Started %s pid %d", id, pid) + } + } } diff --git a/pkg/containerd/etc/init.d/020-containerd b/pkg/containerd/etc/init.d/020-containerd index af34e26da..845a09b1b 100755 --- a/pkg/containerd/etc/init.d/020-containerd +++ b/pkg/containerd/etc/init.d/020-containerd @@ -1,25 +1,14 @@ #!/bin/sh -# bring up containerd -printf "\nStarting containerd\n" -/usr/bin/containerd & - -# wait for socket to be there -while [ ! -S /run/containerd/containerd.sock ] -do - sleep 0.1 -done - # start service containers -service system-init - if [ -d /containers/services ] then for f in $(find /containers/services -mindepth 1 -maxdepth 1 | sort) do /bin/mount --bind "$f/rootfs" "$f/rootfs" mount -o remount,rw "$f/rootfs" - service start "$(basename $f)" done fi + +service system-init