FC: log out the firecracker's console when debug enabled

For shimv2 case, when hypervisor's debug option set, log out
the firecracker's console output which contains the kernel boot
logs; thus it would be easy for system panic debugging.

When agent debug was enabled by passing "agent.log=debug" to
kernel parameter, it will also log out the agent logs from
the console output.

Fixes: #2201

Signed-off-by: lifupan <lifupan@gmail.com>

Gopkg.lock

@@ -89,7 +89,6 @@
   revision = "c4b9ac5c7601384c965b9646fc515884e091ebb9"
 
 [[projects]]
-  branch = "master"
   digest = "1:da4daad2ec1737eec4ebeeed7afedb631711f96bbac0c361a17a4d0369d00c6d"
   name = "github.com/containerd/console"
   packages = ["."]
@@ -707,6 +706,7 @@
     "github.com/BurntSushi/toml",
     "github.com/blang/semver",
     "github.com/containerd/cgroups",
+    "github.com/containerd/console",
     "github.com/containerd/containerd/api/events",
     "github.com/containerd/containerd/api/types",
     "github.com/containerd/containerd/api/types/task",

Gopkg.toml

@@ -78,6 +78,10 @@
   branch = "master"
   name = "github.com/hashicorp/yamux"
 
+[[constraint]]
+  revision = "0650fd9eeb50bab4fc99dceb9f2e14cf58f36e7f"
+  name = "github.com/containerd/console"
+
 [prune]
   non-go = true
   go-tests = true

virtcontainers/fc.go

@@ -6,8 +6,10 @@
 package virtcontainers
 
 import (
+	"bufio"
 	"context"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"os"
@@ -31,6 +33,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/blang/semver"
+	"github.com/containerd/console"
 	"github.com/kata-containers/runtime/virtcontainers/device/config"
 	fcmodels "github.com/kata-containers/runtime/virtcontainers/pkg/firecracker/client/models"
 	"github.com/kata-containers/runtime/virtcontainers/store"
@@ -77,16 +80,12 @@ var fcKernelParams = append(commonVirtioblkKernelRootParams, []Param{
 	{"reboot", "k"},
 	{"panic", "1"},
 	{"iommu", "off"},
-	{"8250.nr_uarts", "0"},
 	{"net.ifnames", "0"},
 	{"random.trust_cpu", "on"},
 
 	// Firecracker doesn't support ACPI
 	// Fix kernel error "ACPI BIOS Error (bug)"
 	{"acpi", "off"},
-
-	// Tell agent where to send the logs
-	{"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)},
 }...)
 
 func (s vmmState) String() string {
@@ -388,13 +387,17 @@ func (fc *firecracker) fcInit(timeout int) error {
 	var args []string
 	var cmd *exec.Cmd
 
+	if !fc.config.Debug && fc.stateful {
+		args = append(args, "--daemonize")
+	}
+
 	//https://github.com/firecracker-microvm/firecracker/blob/master/docs/jailer.md#jailer-usage
 	//--seccomp-level specifies whether seccomp filters should be installed and how restrictive they should be. Possible values are:
 	//0 : disabled.
 	//1 : basic filtering. This prohibits syscalls not whitelisted by Firecracker.
 	//2 (default): advanced filtering. This adds further checks on some of the parameters of the allowed syscalls.
 	if fc.jailed {
-		args = []string{
+		args = append(args,
 			"--id", fc.id,
 			"--node", "0", //FIXME: Comprehend NUMA topology or explicit ignore
 			"--seccomp-level", "2",
@@ -402,8 +405,7 @@ func (fc *firecracker) fcInit(timeout int) error {
 			"--uid", "0", //https://github.com/kata-containers/runtime/issues/1869
 			"--gid", "0",
 			"--chroot-base-dir", fc.chrootBaseDir,
-			"--daemonize",
+		)
-		}
 		if fc.netNSPath != "" {
 			args = append(args, "--netns", fc.netNSPath)
 		}
@@ -414,6 +416,16 @@ func (fc *firecracker) fcInit(timeout int) error {
 
 	}
 
+	if fc.config.Debug && fc.stateful {
+		stdin, err := fc.watchConsole()
+		if err != nil {
+			return err
+		}
+
+		cmd.Stderr = stdin
+		cmd.Stdout = stdin
+	}
+
 	fc.Logger().WithField("hypervisor args", args).Debug()
 	fc.Logger().WithField("hypervisor cmd", cmd).Debug()
 	if err := cmd.Start(); err != nil {
@@ -662,6 +674,16 @@ func (fc *firecracker) startSandbox(timeout int) error {
 		return err
 	}
 
+	if fc.config.Debug && fc.stateful {
+		fcKernelParams = append(fcKernelParams, Param{"console", "ttyS0"})
+	} else {
+		fcKernelParams = append(fcKernelParams, []Param{
+			{"8250.nr_uarts", "0"},
+			// Tell agent where to send the logs
+			{"agent.log_vport", fmt.Sprintf("%d", vSockLogsPort)},
+		}...)
+	}
+
 	kernelParams := append(fc.config.KernelParams, fcKernelParams...)
 	strParams := SerializeParams(kernelParams, "=")
 	formattedParams := strings.Join(strParams, " ")
@@ -1099,3 +1121,37 @@ func (fc *firecracker) generateSocket(id string, useVsock bool) (interface{}, er
 		Port:    uint32(vSockPort),
 	}, nil
 }
+
+func (fc *firecracker) watchConsole() (*os.File, error) {
+	master, slave, err := console.NewPty()
+	if err != nil {
+		fc.Logger().WithField("Error create pseudo tty", err).Debug()
+		return nil, err
+	}
+
+	stdio, err := os.OpenFile(slave, syscall.O_RDWR, 0700)
+	if err != nil {
+		fc.Logger().WithError(err).Debugf("open pseudo tty %s", slave)
+		return nil, err
+	}
+
+	go func() {
+		scanner := bufio.NewScanner(master)
+		for scanner.Scan() {
+			fc.Logger().WithFields(logrus.Fields{
+				"sandbox":   fc.id,
+				"vmconsole": scanner.Text(),
+			}).Infof("reading guest console")
+		}
+
+		if err := scanner.Err(); err != nil {
+			if err == io.EOF {
+				fc.Logger().Info("console watcher quits")
+			} else {
+				fc.Logger().WithError(err).Error("Failed to read guest console")
+			}
+		}
+	}()
+
+	return stdio, nil
+}